diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efbf921ca..c440e4ce0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,7 +204,7 @@ jobs: path: wheelhouse key: >- native-wheel-v1-${{ matrix.id }}-${{ hashFiles('native/**', '!native/tests/**', - '!native/benchmarks/**', '!native/**/*.md') }} + '!native/**/*.md') }} - if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'Linux' name: Install the native toolchain run: | @@ -563,7 +563,7 @@ jobs: - name: Install the native toolchain run: | sudo apt-get update - sudo apt-get install --yes clang-tidy cmake g++-14 libfftw3-dev ninja-build pkg-config + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config - run: uv python install 3.14 - run: uv sync --locked --no-install-package solprop-native - run: uv run poe check-native @@ -608,7 +608,6 @@ jobs: cmake --preset cuda \ -DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_CUDA_HOST_COMPILER=g++-14 \ -DCUDA_ROOT=/usr/local/cuda-12.8 \ - -DSOLPROP_NATIVE_BUILD_BENCHMARKS=OFF \ -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" - name: Compile, tests included working-directory: native @@ -656,7 +655,6 @@ jobs: cmake --preset hip \ -DCMAKE_CXX_COMPILER="$ROCM_PATH/bin/hipcc" -DCMAKE_PREFIX_PATH="$ROCM_PATH" \ -DCMAKE_CXX_SCAN_FOR_MODULES=OFF -DKokkos_ENABLE_ROCTHRUST=OFF \ - -DSOLPROP_NATIVE_BUILD_BENCHMARKS=OFF \ -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" - name: Compile, tests included working-directory: native diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 0b59c4fc6..b756e2129 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -5,7 +5,8 @@ name: NVIDIA CUDA validation # Manual workflow for a self-hosted Linux runner labelled nvidia-gpu, with an NVIDIA -# GPU, the CUDA Toolkit (nvcc, cuFFT, cuBLAS, Compute Sanitizer) and GCC 13 or newer. +# GPU, the CUDA Toolkit 12.8 or newer (nvcc, cuFFT, cuBLAS, Compute Sanitizer) and GCC 14 +# or newer (CMakeLists.txt refuses an older host compiler). # Set the repository variable SOLPROP_NATIVE_CUDA_ARCH to the runner's Kokkos # architecture (for example AMPERE80, ADA89 or HOPPER90); HOPPER90 is used when unset. # The repository is public: keep this workflow dispatch-only and never let the @@ -58,3 +59,5 @@ jobs: run: | compute-sanitizer --tool memcheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py compute-sanitizer --tool racecheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py + compute-sanitizer --tool initcheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py + compute-sanitizer --tool synccheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py diff --git a/.github/workflows/native-deep.yml b/.github/workflows/native-deep.yml new file mode 100644 index 000000000..5795b3a64 --- /dev/null +++ b/.github/workflows/native-deep.yml @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +name: Native deep checks + +# Weekly, by hand, and on a pull request that changes this file: the native checks too +# slow for every pull request. Each job reports (an artifact and the job summary) and +# gates nothing. +# +# - coverage: Clang source coverage of the native code over the GoogleTest suite and +# the native pytest suite (SOLPROP_NATIVE_ENABLE_COVERAGE). Its uncovered lines are +# where to look for dead code. +# - extension-asan: the extension itself built with ASan and UBSan and run under the +# native pytest suite, which exercises the nanobind bindings the gate's sanitizer +# stage (GoogleTest only) never reaches. +# - include-cleaner: clang-tidy's misc-include-cleaner over the sources, as a report of +# missing and unused includes. + +on: + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + # A pull request that changes this workflow runs it, so that a change is seen to work + # before the next weekly run (dispatch needs the workflow on the default branch). + pull_request: + paths: + - .github/workflows/native-deep.yml + +permissions: + contents: read + +concurrency: + group: native-deep-${{ github.ref }} + cancel-in-progress: true + +jobs: + coverage: + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CC: clang-18 + CXX: clang++-18 + OMP_PROC_BIND: "false" + SOLPROP_NATIVE_THREADS: "2" + UV_NO_SYNC: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install Clang, its OpenMP runtime and FFTW + run: | + sudo apt-get update + sudo apt-get install --yes clang-18 llvm-18 libomp-18-dev cmake libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Build the instrumented extension and native tests + # FFTW's pthreads library: Ubuntu's OpenMP FFTW links GNU OpenMP, and Clang's + # build runs LLVM's. No vendor BLAS (the reference profile). + # Report-only: a warning from this job's compiler, which no pull-request lane + # builds with, does not stop the report. + env: + CMAKE_ARGS: >- + -DSOLPROP_NATIVE_WARNINGS_AS_ERRORS=OFF + -DSOLPROP_NATIVE_ENABLE_COVERAGE=ON + -DSOLPROP_NATIVE_PROFILE=REFERENCE + -DSOLPROP_NATIVE_FFTW_THREADING=PTHREADS + -DBUILD_TESTING=ON + run: | + uv pip install --no-deps --no-build-isolation -e native \ + --config-settings build-dir=build/coverage + - name: GoogleTest suite + continue-on-error: true + env: + LLVM_PROFILE_FILE: ${{ github.workspace }}/profiles/gtest-%p-%m.profraw + run: native/build/coverage/solprop_native_tests + - name: Native pytest suite + continue-on-error: true + working-directory: native + env: + LLVM_PROFILE_FILE: ${{ github.workspace }}/profiles/pytest-%p-%m.profraw + run: uv run pytest -n 2 -p no:cacheprovider + - name: Report + if: always() + run: | + extension=$(uv run python -c "import solprop_native._core as m; print(m.__file__)") + llvm-profdata-18 merge -sparse profiles/*.profraw -o coverage.profdata + objects=("$extension" -object native/build/coverage/solprop_native_tests) + sources=(native/src native/include/solprop_native) + mkdir -p native-coverage + llvm-cov-18 report "${objects[@]}" -instr-profile coverage.profdata "${sources[@]}" \ + | tee native-coverage/report.txt + llvm-cov-18 show "${objects[@]}" -instr-profile coverage.profdata "${sources[@]}" \ + -format=html -show-line-counts-or-regions -output-dir native-coverage/html + { echo '## Native source coverage'; echo '```'; tail -n 40 native-coverage/report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-coverage + path: native-coverage + retention-days: 30 + + extension-asan: + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CXX: g++-14 + OMP_PROC_BIND: "false" + SOLPROP_NATIVE_THREADS: "2" + UV_NO_SYNC: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install the native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Build the extension with ASan, UBSan and Kokkos bounds checks + env: + CMAKE_ARGS: >- + -DSOLPROP_NATIVE_WARNINGS_AS_ERRORS=OFF + -DSOLPROP_NATIVE_ENABLE_SANITIZERS=ON + -DSOLPROP_NATIVE_PROFILE=REFERENCE + -DKokkos_ENABLE_DEBUG_BOUNDS_CHECK=ON + run: | + uv pip install --no-deps --no-build-isolation -e native \ + --config-settings build-dir=build/asan \ + --config-settings cmake.build-type=RelWithDebInfo + - name: Native pytest suite under the sanitizers + working-directory: native + # Python is not instrumented, so the ASan runtime is preloaded, with libstdc++ + # beside it: ASan resolves __cxa_throw when it starts, and Python loads no C++ + # runtime of its own, so without it every C++ exception aborts. CPython's own + # allocations at exit are not leaks of this package. setarch -R: Ubuntu's + # high-entropy ASLR can map a library inside ASan's shadow range. Report-only: + # the summary records the outcome and the log is the artifact. + run: | + set -o pipefail + export LD_PRELOAD="$(g++-14 -print-file-name=libasan.so):$(g++-14 -print-file-name=libstdc++.so)" + export ASAN_OPTIONS=detect_leaks=0:abort_on_error=1 + export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 + status=0 + setarch "$(uname -m)" -R uv run pytest -n 2 -p no:cacheprovider 2>&1 | tee ../asan.log || status=$? + { echo '## Native pytest under ASan and UBSan'; echo "exit status ${status}"; echo '```' + grep -E 'ERROR: AddressSanitizer|runtime error|^FAILED|passed|failed' ../asan.log | tail -n 40 + echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-extension-asan + path: asan.log + retention-days: 30 + + include-cleaner: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + CXX: g++-14 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install the native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Configure the cpu-tidy preset (compile commands and generated headers) + working-directory: native + run: | + cmake --preset cpu-tidy -DCMAKE_CXX_COMPILER="$(command -v g++-14)" \ + -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" + # SLEEF's header comes from its own build. + cmake --build --preset cpu-tidy --target solprop_native_sleef_build + - name: misc-include-cleaner + run: | + build=native/build/presets/cpu-tidy + # The pinned clang-tidy and the arguments the build gives it. + mapfile -t tidy < "$build/clang_tidy_command.txt" + git ls-files -z -- 'native/src/*.cpp' 'native/tests/native/*.cpp' | + xargs -0 -n 1 -P "$(nproc)" "${tidy[@]}" --quiet -p "$build" \ + --checks='-*,misc-include-cleaner' --warnings-as-errors='' \ + > include-cleaner.txt 2>&1 || true + findings=$(grep -c 'misc-include-cleaner' include-cleaner.txt || true) + echo "## misc-include-cleaner: ${findings} findings" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-include-cleaner + path: include-cleaner.txt + retention-days: 30 diff --git a/native/benchmarks/size_cliffs.py b/benchmarks/size_cliffs.py similarity index 96% rename from native/benchmarks/size_cliffs.py rename to benchmarks/size_cliffs.py index 873f28661..1504f4088 100644 --- a/native/benchmarks/size_cliffs.py +++ b/benchmarks/size_cliffs.py @@ -17,7 +17,7 @@ cheapest, because fixed per-evaluation costs make small grids dearer per sample. ``--check`` exits non-zero on one. Run on a quiet machine, with the threads pinned:: - OMP_PLACES=cores OMP_PROC_BIND=close python native/benchmarks/size_cliffs.py --check + OMP_PLACES=cores OMP_PROC_BIND=close python -m benchmarks.size_cliffs --check """ from __future__ import annotations @@ -31,6 +31,10 @@ import warnings from typing import Any, cast +from solprop.driver import Simulation +from solprop.grid import FieldGrid +from solprop.stepper import RungeKutta + FACTORS = (1.0, 1.125, 1.25, 1.375, 1.5, 1.75, 2.0) LENGTH = 1.5 STEP = LENGTH / 4000 @@ -39,10 +43,6 @@ def simulation(factor: float, fft_size: str, backend: str, steps: int) -> Any: """Return the RDW case on one grid, with `steps` capped Runge-Kutta steps.""" - from solprop.driver import Simulation - from solprop.grid import FieldGrid - from solprop.stepper import RungeKutta - grid = FieldGrid.from_limits( LENGTH, 800e-9, (90e-9, 4e-6), factor * 400e-15, fft_size=fft_size ) diff --git a/native/benchmarks/statistics_cost.py b/benchmarks/statistics_cost.py similarity index 69% rename from native/benchmarks/statistics_cost.py rename to benchmarks/statistics_cost.py index 93f2d1487..a79affa5d 100644 --- a/native/benchmarks/statistics_cost.py +++ b/benchmarks/statistics_cost.py @@ -2,7 +2,18 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Measure the periodic statistics channel inside propagation.""" +"""Measure the periodic statistics channel inside propagation. + +Each case runs three ways through the backend's session, interleaved round by round: +``plain`` (no periodic statistics), ``stashed`` (the electron density's plasma stash +alone) and ``watched`` (every periodic row). The JSON line reports what the channel and +its collection cost per step and per row. ``m2_reduced`` is the two-mode helium RDW +capillary and ``ctc_reduced`` the one-mode hydrogen comb-to-continuum capillary below, +both at rtol 1e-6 and atol 1e-10: the workloads of the native package's former +``statistics_cost`` harness, reproduced exactly:: + + python -m benchmarks.statistics_cost --case m2_reduced --repeats 5 +""" from __future__ import annotations @@ -13,7 +24,17 @@ import os import time import warnings -from typing import TypedDict, cast +from typing import TypedDict + +import numpy as np + +from solprop.backends.registry import select +from solprop.backends.session import Start +from solprop.backends.sessions import start_session +from solprop.driver import Simulation +from solprop.exceptions import MaterialRangeWarning +from solprop.stats import StatsSpec +from solprop.translate.run_request import RunRequest PERIODIC = ("energy", "peak_power", "fwhm_t", "electron_density") @@ -26,41 +47,43 @@ class Arm(TypedDict): rows: int -class CaseTable(TypedDict): - """Geometry and sampling values of the M2 benchmark cases (``m2_cases.py``).""" - - radius: float - length: float - pressure: float | tuple[float, float] - modes: int - lambda_min: float - lambda_max: float - trange: float - n_save: int - expected_steps: int - +M2_REDUCED = { + "radius": 125e-6, + "length": 1.0, + "gas": "He", + "pressure": (1.2, 0.0), + "modes": 2, + "wavelength": 800e-9, + "wavelength_limits": (100e-9, 1.5e-6), + "trange": 200e-15, + "fwhm": 7.5e-15, + "energy": 275e-6, + "n_save": 51, +} +"""The reduced resonant-dispersive-wave capillary: the ``rdw`` case's 7.5 fs, 275 uJ +pulse at 800 nm in helium, over 1 m at 1.2 bar falling to vacuum, two modes.""" -class FamilyTable(TypedDict): - """Pulse and gas values of the M2 benchmark cases (``m2_cases.py``).""" +CTC_REDUCED = { + "radius": 16e-6, + "length": 0.1, + "gas": "H2", + "pressure": 40.0, + "modes": 1, + "wavelength": 515e-9, + "wavelength_limits": (250e-9, 2e-6), + "trange": 2e-12, + "fwhm": 400e-15, + "energy": 10e-6, + "n_save": 21, +} +"""The reduced comb-to-continuum capillary: 400 fs, 10 uJ at 515 nm in 40 bar of +hydrogen, lossless, with Raman, over 0.1 m in one mode.""" - gas: str - wavelength: float - tau_fwhm: float - energy: float - loss: bool - -def _case_tables() -> tuple[CaseTable, FamilyTable]: - from m2_cases import CASES, FAMILIES - - return cast(CaseTable, CASES["m2_reduced"]), cast( - FamilyTable, FAMILIES["m2_reduced"] - ) - - -def _worker( +def _worker( # noqa: PLR0915 - one measurement: set up, run the arms, report backend_name: str, case_name: str, + *, target: float, repeats: int, modes: int | None, @@ -71,7 +94,7 @@ def _worker( fftw_threads = 1 native_threads = int(os.environ.get("SOLPROP_NATIVE_THREADS", "1")) if backend_name == "native": - import solprop_native + import solprop_native # noqa: PLC0415 # pyright: ignore[reportMissingImports] build = solprop_native.build_info() blas_backend = build.blas_backend @@ -84,36 +107,43 @@ def _worker( else: blas_threads = "disabled" - import numpy as np - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.driver import Simulation - from solprop.exceptions import MaterialRangeWarning - from solprop.stats import StatsSpec - from solprop.translate.run_request import RunRequest - with warnings.catch_warnings(): warnings.simplefilter("ignore", MaterialRangeWarning) if case_name == "ctc_reduced": - from ctc import build_simulation - - simulation = build_simulation(case_name, backend_name) + case = CTC_REDUCED + simulation = Simulation.from_capillary( + case["radius"], + case["length"], + case["gas"], + case["pressure"], + wavelength=case["wavelength"], + wavelength_limits=case["wavelength_limits"], + trange=case["trange"], + fwhm=case["fwhm"], + energy=case["energy"], + modes=case["modes"], + loss=False, + raman=True, + n_save=case["n_save"], + rtol=1e-6, + atol=1e-10, + backend=backend_name, + ) else: - case, family = _case_tables() + case = M2_REDUCED simulation = Simulation.from_capillary( case["radius"], case["length"], - family["gas"], + case["gas"], case["pressure"], pressure_floor=0.0, - wavelength=family["wavelength"], - wavelength_limits=(case["lambda_min"], case["lambda_max"]), + wavelength=case["wavelength"], + wavelength_limits=case["wavelength_limits"], trange=case["trange"], - fwhm=family["tau_fwhm"], - energy=family["energy"], + fwhm=case["fwhm"], + energy=case["energy"], modes=case["modes"] if modes is None else modes, - loss=family["loss"], + loss=True, n_save=case["n_save"], rtol=1e-6, atol=1e-10, @@ -249,10 +279,10 @@ def main() -> None: _worker( arguments.backend, arguments.case, - arguments.target, - arguments.repeats, - arguments.modes, - arguments.period, + target=arguments.target, + repeats=arguments.repeats, + modes=arguments.modes, + period=arguments.period, ) diff --git a/benchmarks/worker.py b/benchmarks/worker.py index 8e5d694bf..43be0bfe9 100644 --- a/benchmarks/worker.py +++ b/benchmarks/worker.py @@ -71,7 +71,7 @@ def emit(kind: str, **values: object) -> None: print(canonical({"event": kind, **values}), flush=True) -HOST_SPACES = ("OpenMP", "Threads", "Serial") +HOST_SPACES = ("OpenMP", "Serial") """Kokkos execution spaces of a native CPU build; any other is an accelerator's.""" diff --git a/changelog.d/native-build-options.removed.md b/changelog.d/native-build-options.removed.md new file mode 100644 index 000000000..fed266081 --- /dev/null +++ b/changelog.d/native-build-options.removed.md @@ -0,0 +1 @@ +Building solprop-native from source: removed the `SOLPROP_NATIVE_GEMM_PROVIDER` option (every build uses the hybrid provider, `build_info().gemm_provider` is always `"HYBRID"`), the `cuda-custom`, `cuda-native` and `cuda-tpl` presets, `SOLPROP_NATIVE_BUILD_BENCHMARKS` and its GEMM microbenchmark, `SOLPROP_NATIVE_ENABLE_THREADS` (the Kokkos Threads host space; CPU builds run OpenMP) and IntelLLVM (`icpx`) support (Linux CPU builds take GCC 14 or newer, or Clang 14 or newer). A CPU build no longer needs a single-precision FFTW, and the wheels no longer carry one. diff --git a/changelog.d/native-tooling.changed.md b/changelog.d/native-tooling.changed.md new file mode 100644 index 000000000..7fee63b7c --- /dev/null +++ b/changelog.d/native-tooling.changed.md @@ -0,0 +1 @@ +The native C++ checks: every tracked C++ file is formatted, clang-tidy is pinned to LLVM 23.1 and enables whole check families, the sources compile with stronger warnings, the sanitizer build stops at the first report and checks Kokkos view bounds, and `uv run poe fmt-native` and `uv run poe tidy-native` format and tidy native files locally. diff --git a/docs/dev/backends.md b/docs/dev/backends.md index da62d8d9f..caffe435a 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -762,13 +762,13 @@ mask and the in-stage absorber preserve it, and the measured saving are `nsz`, approximate functions and reciprocals, and flush-to-zero forbidden). On `native`, as `native/AGENTS.md` and `native/CMakeLists.txt` set it: - **Contraction.** A CPU build compiles with `-ffp-contract=fast` under GCC and - `-ffp-contract=fast-honor-pragmas` under Clang and `icpx` (plain `fast` makes Clang + `-ffp-contract=fast-honor-pragmas` under Clang (plain `fast` makes Clang ignore the pragmas), so the compiler may fuse any multiply and add into one rounding. Its one hazard is exact cancellation (`x*y - y*x` need not be zero). `build_info().optimization_policy` records the flag. - **Strict blocks.** A protected function or lambda carries `SOLPROP_NATIVE_STRICT_FUNCTION` and starts its body with `SOLPROP_NATIVE_STRICT_BLOCK` - (`include/solprop_native/fp.hpp`): under Clang and `icpx` a `#pragma clang fp + (`include/solprop_native/fp.hpp`): under Clang a `#pragma clang fp contract(off)`, which survives inlining; under GCC, which ignores the pragma, `optimize("fp-contract=off")` on the function, so a protected helper is a call there. Both settings are lexical, and `Kokkos::complex`'s operators keep their header's diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index b43d5cde4..0c2a23454 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -508,7 +508,7 @@ against 2.40 s), where it was 1.4 times. (smooth-size-cliffs)= ### Smooth grid sizes -`native/benchmarks/size_cliffs.py` times the four-mode RDW capillary per source +`python -m benchmarks.size_cliffs` times the four-mode RDW capillary per source evaluation (short and long capped Runge-Kutta windows, their difference) on 2-3-5-smooth and power-of-two grids, and flags a size whose cost per sample exceeds its nearest power of two's by more than 25 %. Two size boundaries made cliffs: the stage threshold (8192 diff --git a/docs/dev/dependencies.md b/docs/dev/dependencies.md index 461700234..1ea7326b4 100644 --- a/docs/dev/dependencies.md +++ b/docs/dev/dependencies.md @@ -119,6 +119,13 @@ drawsvg (the structure figures of `docs/dev/figures/build.py` and the geometry f `docs/guide/figures/geometries.py` only, run by hand with `poe figures`; the SVGs it writes are committed, so the documentation builds without it). +The native gate's C++ tools are pinned exactly in the dev group, so that its checks do +not depend on a machine's LLVM: **clang-format 23.1.0** (`poe fmt-native` and the lint +stage) and **clang-tidy 23.1.4** (`poe tidy-native` and the tidy stage; its PyPI wheels +start at 23.1.3, the nearest to the formatter's release). clang-tidy is installed on +Linux and macOS only: the gate that runs it is bash, and it has no PyPI wheel for +Windows on Arm, where a sync would build LLVM from source. + ## Candidates (not approved; only evaluate for a selected use case) | package | for | status | diff --git a/docs/dev/tooling.md b/docs/dev/tooling.md index fa46bc269..ac7c361c2 100644 --- a/docs/dev/tooling.md +++ b/docs/dev/tooling.md @@ -159,6 +159,37 @@ ruff (check with `--fix`, then format), numpydoc validation, codespell, `uv-lock `reuse lint-file` on the staged files. Check the `rev` pins against current releases when updating. +## Native C++ tooling + +The native backend's C++ (`native/`) has its own tools, judged against this project: a +double-precision Kokkos code with one process-wide runtime, no untrusted input, a +Python oracle for every result, and end-to-end runs as the measure of speed. + +| tool | status | where, or why not | +|---|---|---| +| clang-format (pinned 23.1.0) | adopted | every tracked C++ file, in `check-native`'s lint stage; `poe fmt-native` formats in place | +| clang-tidy (pinned 23.1.4), whole families | adopted | `bugprone`, `performance`, `modernize`, `misc`, `readability`, `clang-analyzer` and `cppcoreguidelines-special-member-functions`, each exclusion stated in `native/.clang-tidy`; warnings are errors; the tidy stage; `poe tidy-native` for changed files | +| compiler warnings | adopted | `-Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow -Wundef -Wnon-virtual-dtor -Woverloaded-virtual`, with Clang's `-Wimplicit-fallthrough` and GCC's `-Wduplicated-cond -Wduplicated-branches -Wlogical-op`; errors in development and CI builds | +| ASan + UBSan with Kokkos bounds checks | adopted | the `cpu-sanitize` preset over the GoogleTest suite, UBSan stopping at its first report | +| compute-sanitizer (memcheck, racecheck, initcheck, synccheck) | adopted | `.github/workflows/cuda.yml`, on a device | +| Clang source coverage | scheduled | `.github/workflows/native-deep.yml`, weekly: GoogleTest and native pytest, a report whose zero-count lines are the dead-code candidates | +| ASan + UBSan on the extension | scheduled | `native-deep.yml`: the native pytest suite through the bindings | +| `misc-include-cleaner` | scheduled | `native-deep.yml`, a report; too noisy across Kokkos's umbrella headers to gate | +| perf, Hotspot, Instruments, Nsight Systems and Compute, rocprof, optimisation remarks, llvm-mca, Compiler Explorer | ad hoc | profiling and code-generation questions, by hand; the deciding number is `python -m benchmarks ab` | +| cppcheck | rejected | overlaps clang-tidy and misreads Kokkos lambdas | +| include-what-you-use (the binary) | rejected | `misc-include-cleaner` gives the same report from the pinned clang-tidy | +| CodeChecker, scan-build | rejected | front ends to the Clang static analyser, which `clang-analyzer-*` runs | +| GCC `-fanalyzer` | rejected | C-oriented and noisy on C++ templates; `clang-analyzer-*` covers it | +| CodeQL, SonarQube | rejected | security analysis of untrusted input, which this code has none of | +| ThreadSanitizer | rejected for the gate | needs an instrumented libomp, FFTW and BLAS; the thread-invariance probes cover the outcomes. An on-demand campaign only | +| MemorySanitizer, TypeSanitizer | rejected | MSan needs every library instrumented; TySan is experimental and strict aliasing is not a risk here | +| Valgrind | rejected | ASan covers it, much faster, and it does not run on Apple silicon | +| Clang thread-safety annotations | rejected | there is one mutex | +| Google Benchmark | rejected | microbenchmarks mislead here; the deciding number is the end-to-end `benchmarks ab` | +| Tracy, HPCToolkit | rejected | Kokkos Tools and the profilers above cover them | +| fuzzers (libFuzzer, RapidCheck) | rejected | the seam's inputs are generated by Hypothesis in the Python tests | +| MPFR | rejected | mpmath oracles in the Python tests give the high-precision references | + ## Sphinx `docs/conf.py`: myst-parser, numpydoc (without napoleon), autosummary (recursive, `generate = True`), diff --git a/docs/guide/performance/building-native.md b/docs/guide/performance/building-native.md index bf3444f02..88e778c77 100644 --- a/docs/guide/performance/building-native.md +++ b/docs/guide/performance/building-native.md @@ -26,11 +26,11 @@ the arithmetic is double precision throughout, and no build enables fast-math. | CMake | 3.23 or newer (with build isolation, scikit-build-core fetches one from PyPI if none is found) | | build tool | Ninja is recommended; scikit-build-core uses it when available | | C++20 compiler, macOS | Clang 14 or newer; Homebrew LLVM is recommended and selected by default | -| C++20 compiler, Linux CPU | GCC 14 or newer, IntelLLVM (`icpx`) 2022 or newer, or Clang 14 or newer / AMD AOCC | +| C++20 compiler, Linux CPU | GCC 14 or newer, or Clang 14 or newer / AMD AOCC | | C++20 compiler, CUDA | GCC 14 or newer as the host compiler, with `nvcc` from CUDA 12.8 or newer (qualified on CUDA 13) | | C++20 compiler, HIP | ROCm's Clang (`hipcc` or `amdclang++`) | | OpenMP (CPU builds) | the compiler's runtime; on macOS Homebrew's `libomp` | -| FFT (CPU builds) | FFTW 3 with its threaded libraries, in double and single precision (or MKL's FFTW interface) | +| FFT (CPU builds) | FFTW 3 in double precision with its threaded libraries (or MKL's FFTW interface); the build needs no single-precision FFTW | | BLAS (CPU builds) | optional but recommended: Accelerate (macOS), OpenBLAS, MKL or another LP64 BLAS | | GPU libraries | cuFFT and cuBLAS (CUDA), or rocFFT and rocBLAS (HIP) | | network | configuring fetches the pinned dependencies below from GitHub (see [offline builds](#offline-builds)) | @@ -49,8 +49,7 @@ them: The Python build requirements are scikit-build-core 0.11 and nanobind 2.9 or newer. CPU builds contract multiply-adds (`-ffp-contract=fast` with GCC, -`-ffp-contract=fast-honor-pragmas` with Clang and `icpx`, after `-fno-fast-math` and -`-fp-model=precise`) except in a few protected blocks, as +`-ffp-contract=fast-honor-pragmas` with Clang, after `-fno-fast-math`) except in a few protected blocks, as [SEP 25](../../dev/seps/archive/sep-0025-rounding-level-reproducibility.md) sets out; `build_info().compiler_flags` and `optimization_policy` record the setting. GPU builds follow their compilers' defaults throughout ([GPU builds](#gpu-builds)). @@ -82,7 +81,7 @@ export CMAKE_ARGS="-DCMAKE_OSX_SYSROOT=$SDKROOT" ``` For an Apple silicon performance build, the pinned dependency recipe enables FFTW -3.3.11's NEON codelets in **both precisions**. Compiling scalar code with `-mcpu` +3.3.11's NEON codelets. Compiling scalar code with `-mcpu` does not enable those codelets. The recipe also retains the Arm cycle counter needed by measured planning, the Apple M1 distribution baseline and the macOS 14 deployment target. Its FFTW threading libraries and the native extension share one source-built @@ -101,9 +100,8 @@ uv run --no-sync python -m solprop_native Check the reported `fft_library` and `openmp_library` before benchmarking. The [pinned upstream configuration](https://github.com/FFTW/fftw3/blob/fftw-3.3.11/configure.ac) -supports `--enable-neon` for double and single precision on AArch64. The installed -libraries identify their codelet family in `fftw_version` and `fftwf_version`, which -both end in `-neon` for this recipe. Keep these prefixes separate from a comparison's +supports `--enable-neon` on AArch64. The installed library identifies its codelet family +in `fftw_version`, which ends in `-neon` for this recipe. Keep these prefixes separate from a comparison's baseline libraries. See [OpenMP runtimes](#openmp-runtimes) below for the limits of mixing independently bundled wheels. @@ -124,7 +122,7 @@ cd solprop CXX=g++-14 uv pip install --python /path/to/env/bin/python . ./native ``` -`libfftw3-dev` provides the OpenMP and pthreads FFTW libraries in both precisions. For +`libfftw3-dev` provides the OpenMP and pthreads FFTW libraries. For MKL instead of OpenBLAS, source oneAPI first (`source /opt/intel/oneapi/setvars.sh`): a set `MKLROOT` makes the automatic search choose MKL. Other compilers and libraries are in [Linux compilers and libraries](#linux-compilers-and-libraries). @@ -226,7 +224,7 @@ hand through `CXXFLAGS` appear in `compiler_flags`. sources under `native/` are not the ones the extension was compiled from (it names `uv sync`; a wheel skips this); when the installed solprop lacks a driver member or bundle field the build relies on (it names them); when the build enables fast-math; when the execution-space record is inconsistent; when a GPU build sees no -device; when the GEMM provider is unknown or its BLAS is missing or unidentified; when +device; when the GEMM provider is unknown or its BLAS is unidentified; when a reference build carries a vendor BLAS or machine-specific code, or a distribution build carries native code or no BLAS; and when a GPU build's BLAS is not the device's. Then, in solprop: @@ -305,7 +303,7 @@ order. Keep each profile in its own build directory; switching the profile in on rebuilds everything. Configuring prints one summary line, for example ```text --- solprop-native build profile: local; BLAS Accelerate (automatic: Accelerate (macOS)); GEMM provider HYBRID; native architecture ON; CPU baseline none; IPO ON (Release builds) +-- solprop-native build profile: local; BLAS Accelerate (automatic: Accelerate (macOS)); native architecture ON; CPU baseline none; IPO ON (Release builds) ``` and `build_info()` reports the same choice in `build_profile`, `cpu_baseline`, @@ -356,7 +354,6 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | option | default | values and effect | |---|---|---| | `SOLPROP_NATIVE_ENABLE_OPENMP` | `ON` | the OpenMP host execution space and threaded FFTW. `OFF` in GPU builds, whose host space is Serial | -| `SOLPROP_NATIVE_ENABLE_THREADS` | `OFF` | the Kokkos Threads (C++ threads) host space with FFTW's threads library, for a build that adds no OpenMP runtime (a fallback for Windows builds); requires `SOLPROP_NATIVE_ENABLE_OPENMP=OFF`. Kernels then run at the runtime's thread count, without per-operation sizing | | `SOLPROP_NATIVE_ENABLE_CUDA` | `OFF` | the CUDA execution space with cuFFT (Linux, GNU host compiler) | | `SOLPROP_NATIVE_ENABLE_HIP` | `OFF` | the HIP execution space with rocFFT (Linux, ROCm Clang); exclusive with CUDA | | `SOLPROP_NATIVE_CUDA_ARCH` | `HOPPER90` | the Kokkos architecture suffix, `Kokkos_ARCH_` ([CUDA](#cuda)) | @@ -377,14 +374,13 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | `SOLPROP_NATIVE_BLAS_ROOT` | empty | a prefix searched first for the BLAS | | `SOLPROP_NATIVE_BLAS_LIBRARIES` | empty | an explicit `;`-separated library list (not together with `SOLPROP_NATIVE_BLAS_VENDOR`) | | `SOLPROP_NATIVE_MKL_ROOT` | `$MKLROOT` | where the LP64 MKL libraries are looked for | -| `SOLPROP_NATIVE_GEMM_PROVIDER` | `HYBRID` | `HYBRID` (vendor GEMM for eligible transforms, custom kernels for the rest), `KOKKOSBLAS_TPL` (requires a BLAS), `KOKKOSBLAS_NATIVE` or `CUSTOM`; the alternatives are for comparison and validation | **FFT** | option | default | values and effect | |---|---|---| | `SOLPROP_NATIVE_FFT_PROVIDER` | `FFTW` | `FFTW`, or `MKL_FFTW` (MKL's FFTW3 interface, Linux). Host builds only | -| `SOLPROP_NATIVE_FFTW_THREADING` | `OPENMP` | `OPENMP` (`libfftw3_omp`) or `PTHREADS` (`libfftw3_threads`); IntelLLVM with FFTW requires `PTHREADS` | +| `SOLPROP_NATIVE_FFTW_THREADING` | `OPENMP` | `OPENMP` (`libfftw3_omp`) or `PTHREADS` (`libfftw3_threads`) | | `SOLPROP_NATIVE_RAMAN_FFT_PROVIDER` | `ACCELERATE` on macOS CPU builds, else `KOKKOSFFT` | construction-time Raman selection: with NEON FFTW, vDSP handles short parallel power-of-two rows (up to 16,384 samples); serial and longer rows use Kokkos-FFT. Without NEON, only short serial rows use Kokkos-FFT. `KOKKOSFFT` forces that provider; `ACCELERATE` is macOS CPU only | | `FFTW_ROOT` | the Homebrew prefix on macOS | a non-default FFTW installation | @@ -396,7 +392,6 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | `SOLPROP_NATIVE_WARNINGS_AS_ERRORS` | as `BUILD_TESTING`: `OFF` in pip and uv builds, `ON` in the presets | `-Werror` on the package's own sources. Continuous integration turns it on for its wheels too | | `SOLPROP_NATIVE_ENABLE_CLANG_TIDY` | `OFF` | runs clang-tidy while compiling the package's sources | | `SOLPROP_NATIVE_ENABLE_SANITIZERS` | `OFF` | AddressSanitizer and UBSan on the package's targets; CPU builds only | -| `SOLPROP_NATIVE_BUILD_BENCHMARKS` | `OFF` | builds the `solprop_native_gemm_benchmark` microbenchmark (never installed) | | `SOLPROP_NATIVE_BUNDLED_LICENCES` | the environment variable of that name | a directory of `LICENSE-*.txt` files, installed beside the extension: the licences of the libraries a wheel repair copies in (FFTW, the OpenMP runtime) | The installed module of a local build keeps the directories of a BLAS it linked outside @@ -407,11 +402,12 @@ OpenBLAS is. ### Presets `native/CMakePresets.json` defines development builds, all with `BUILD_TESTING=ON`: -`cpu-debug`, `cpu-tidy` and `cpu-sanitize` (reference profile, Debug), `cpu-benchmark` -(local, Release, IPO and benchmarks), the CUDA builds `cuda` (hybrid GEMM with cuBLAS), -`cuda-tpl`, `cuda-native` and `cuda-custom` (the other GEMM providers, for comparison), -and `hip`. The presets do not choose a compiler: pass `-DCMAKE_CXX_COMPILER=...` (on -macOS, Homebrew's `clang++`) and `-DPython_EXECUTABLE=...`. +`cpu-debug`, `cpu-tidy` and `cpu-sanitize` (reference profile, Debug; `cpu-sanitize` +adds ASan, UBSan that stops at its first report, and Kokkos's view bounds checks), +`cpu-benchmark` (local, Release, IPO), `cuda` (with cuBLAS) and `hip`. Dense transforms +take the vendor GEMM where a transform is eligible and the package's own kernels for the +rest, in every build. The presets do not choose a compiler: pass +`-DCMAKE_CXX_COMPILER=...` (on macOS, Homebrew's `clang++`) and `-DPython_EXECUTABLE=...`. ### Offline builds @@ -461,22 +457,20 @@ pinned whichever is used. | compiler | notes | |---|---| | GCC 14 or newer | the reference route, with GNU OpenMP (`libgomp`). GCC 13 is refused: it compiles the strict floating-point helpers through the stack, and coupled Multistep steps took 1.25 to 1.45 times as long as with GCC 14 | -| IntelLLVM (`icpx`) 2022 or newer | compiled with `-fp-model=precise` (its default is fast floating point) and then `-ffp-contract=fast-honor-pragmas`; requires `SOLPROP_NATIVE_FFTW_THREADING=PTHREADS` with FFTW, because distribution OpenMP FFTW libraries link GNU OpenMP; uses the compiler's own archiver for LTO | | Clang 14 or newer, AMD AOCC | compiled with `-fno-fast-math`, then `-ffp-contract=fast-honor-pragmas` | | provider | how to select it | |---|---| | OpenBLAS | the automatic choice when MKL is not configured; `-DSOLPROP_NATIVE_BLAS_VENDOR=OPENBLAS` forces it, and `-DBLAS_openblas_LIBRARY=/path/to/libopenblas.so` names a specific library | -| MKL BLAS | `source /opt/intel/oneapi/setvars.sh` (sets `MKLROOT`), `-DSOLPROP_NATIVE_MKL_ROOT=...` or `-DSOLPROP_NATIVE_BLAS_VENDOR=MKL`. LP64 interface; GCC links MKL's GNU threading layer, IntelLLVM its Intel layer | +| MKL BLAS | `source /opt/intel/oneapi/setvars.sh` (sets `MKLROOT`), `-DSOLPROP_NATIVE_MKL_ROOT=...` or `-DSOLPROP_NATIVE_BLAS_VENDOR=MKL`. LP64 interface, with MKL's GNU threading layer | | AOCL-BLIS and other BLAS | `-DSOLPROP_NATIVE_BLAS_LIBRARIES="/path/libblis-mt.so;/path/libflame.so"` (BLIS alone lacks complex rotations that Kokkos Kernels' BLAS layer links; AOCL's LP64 libFLAME supplies them and needs `libaoclutils` on the loader path) | | FFTW | the default; `-DFFTW_ROOT=...` for a non-system installation, such as AOCL's FFTW | | MKL FFT | `-DSOLPROP_NATIVE_FFT_PROVIDER=MKL_FFTW`: MKL's FFTW3 interface, independent of the BLAS choice. MKL ignores FFTW's planning flags. Sourcing oneAPI never selects it by itself | **FFTW threading.** `OPENMP` links FFTW's OpenMP library, which must use the same OpenMP runtime as the rest of the build: GNU `libgomp` for distribution packages built with -GCC. `PTHREADS` links FFTW's pthreads library instead; it is required for IntelLLVM and -is the safer choice whenever a second OpenMP runtime could otherwise be loaded. With -pthreads FFTW, start with `OMP_PROC_BIND=false`: a bound OpenMP main thread can confine +GCC. `PTHREADS` links FFTW's pthreads library instead; it is the safer choice whenever +a second OpenMP runtime could otherwise be loaded. With pthreads FFTW, start with `OMP_PROC_BIND=false`: a bound OpenMP main thread can confine the FFTW worker threads it creates to its own core. **OpenBLAS threading.** A pthreads OpenBLAS (the Debian and Ubuntu default) keeps its own @@ -537,8 +531,8 @@ export CMAKE_ARGS="-DSOLPROP_NATIVE_ENABLE_CUDA=ON -DSOLPROP_NATIVE_ENABLE_OPENM uv pip install --python /path/to/env/bin/python . ./native ``` -The `cuda` preset configures the same build for development, with the native tests and -the GEMM benchmark (`cmake --preset cuda -DPython_EXECUTABLE=... -DCUDA_ROOT=...`, from +The `cuda` preset configures the same build for development, with the native tests +(`cmake --preset cuda -DPython_EXECUTABLE=... -DCUDA_ROOT=...`, from `native/`). ### HIP diff --git a/native/.clang-format b/native/.clang-format index acbf22140..15a0947b2 100644 --- a/native/.clang-format +++ b/native/.clang-format @@ -7,3 +7,25 @@ ColumnLimit: 88 DerivePointerAlignment: false PointerAlignment: Left SortIncludes: CaseSensitive +InsertNewlineAtEOF: true +# A NOLINT comment is never reflowed: its reason must stay on the suppressing line. +CommentPragmas: '^ (IWYU pragma:|NOLINT)' +# No RemoveSemicolon: clang-format parses `const auto f = KOKKOS_LAMBDA(...) {...};` as a +# function definition and removes the semicolon the declaration needs. Declaring the +# macro (`Macros:`) fixes that but makes formatting general.cpp minutes slower. +# Include blocks: this package's and the directory's own headers ("..."), then the +# third-party libraries, then the standard library, then platform headers (the operating +# system, the compiler and the device runtimes). An include under #if stays in its own +# block, where it is. +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"' + Priority: 1 + - Regex: '^<(Kokkos|impl/Kokkos|nanobind/|fftw3\.h|sleef\.h|gtest/)' + Priority: 2 + CaseSensitive: true + - Regex: '^<[a-z_]+>$' + Priority: 3 + CaseSensitive: true + - Regex: '^<' + Priority: 4 diff --git a/native/.clang-tidy b/native/.clang-tidy index cca785169..6609abb16 100644 --- a/native/.clang-tidy +++ b/native/.clang-tidy @@ -2,43 +2,93 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# Whole check families. The exclusions, one reason each: +# - bugprone-easily-swappable-parameters: numerical kernels take runs of same-typed +# extents and scalars by design. +# - performance-enum-size: the enums are few and small; a narrower base type gains +# nothing. +# - modernize-use-trailing-return-type: style only; the code uses leading return types. +# - modernize-avoid-c-arrays: small fixed arrays are the idiom in device code. +# - misc-non-private-member-variables-in-classes: plain aggregates (Problem, plans, +# records) are the seam's data. +# - misc-include-cleaner: too noisy across Kokkos's umbrella headers to gate; it runs +# as a weekly report (.github/workflows/native-deep.yml). +# - misc-no-recursion: recursion is a design choice in host code, not a defect. +# - misc-const-correctness: it judges one template instantiation and the active #if +# branch, so it marks const the locals another instantiation's `if constexpr` or a +# device branch assigns (its fixes broke the CPU build in about 30 places). +# - readability-magic-numbers: the numerical constants are cited where they appear. +# - readability-identifier-length: short names follow the mathematics (dz, n, k). +# - readability-math-missing-parentheses: precedence in arithmetic is not ambiguous. +# - readability-uppercase-literal-suffix: style only. +# - readability-trailing-comma: style only, and clang-format owns layout. +# - readability-braces-around-statements: the LLVM convention leaves single +# statements unbraced, and adding braces around bodies with #if is unsafe. +# - readability-function-size, readability-function-cognitive-complexity: the long +# setup functions and hot kernels are split first (a later change), then gated. +# - readability-avoid-nested-conditional-operator: nested conditionals initialise +# const locals, in kernels too; an immediately invoked lambda reads worse. +# - readability-convert-member-functions-to-static: it sees the CPU build only, where +# members such as hybrid_shape_eligible() read no state; their device branches do. +# - readability-use-std-min-max: `if (a > m) m = a` keeps m on a NaN where std::max +# takes the NaN, and std::max is not device code. +# - readability-make-member-function-const: Kokkos views are handles, so a member +# that writes through one would be marked const although it changes the state. +# - clang-analyzer-osx.*: Apple framework checks; the code calls no Objective-C or +# Core Foundation API. +# - clang-analyzer-cplusplus.NewDeleteLeaks: it reports Kokkos's shared allocation +# records (freed through a reference count it cannot follow) as leaks; the code holds +# no raw ownership (native/AGENTS.md), and the ASan builds check for leaks. +# - clang-analyzer-optin.performance.Padding: the cold configuration structs are +# ordered by concern, not by size. Checks: > -*, - clang-analyzer-core.*, - clang-analyzer-deadcode.*, - clang-analyzer-security.*, - bugprone-argument-comment, - bugprone-assert-side-effect, - bugprone-exception-escape, - bugprone-infinite-loop, - bugprone-macro-parentheses, - bugprone-multiple-statement-macro, - bugprone-parent-virtual-call, - bugprone-return-const-ref-from-parameter, - bugprone-sizeof-expression, - bugprone-string-constructor, - bugprone-suspicious-enum-usage, - bugprone-suspicious-memory-comparison, - bugprone-suspicious-realloc-usage, - bugprone-unhandled-exception-at-new, - bugprone-unused-local-non-trivial-variable, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-move-const-arg, - performance-no-automatic-move, - performance-type-promotion-in-math-fn, - performance-unnecessary-copy-initialization, + bugprone-*, + -bugprone-easily-swappable-parameters, + performance-*, + -performance-enum-size, + modernize-*, + -modernize-use-trailing-return-type, + -modernize-avoid-c-arrays, + misc-*, + -misc-non-private-member-variables-in-classes, + -misc-include-cleaner, + -misc-no-recursion, + -misc-const-correctness, + readability-*, + -readability-magic-numbers, + -readability-identifier-length, + -readability-math-missing-parentheses, + -readability-uppercase-literal-suffix, + -readability-trailing-comma, + -readability-braces-around-statements, + -readability-function-size, + -readability-function-cognitive-complexity, + -readability-avoid-nested-conditional-operator, + -readability-make-member-function-const, + -readability-use-std-min-max, + -readability-convert-member-functions-to-static, + cppcoreguidelines-special-member-functions, + clang-analyzer-*, + -clang-analyzer-osx.*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-cplusplus.NewDeleteLeaks, portability-restrict-system-includes, - portability-simd-intrinsics, - modernize-use-equals-default, - modernize-use-equals-delete, - modernize-use-nullptr, - modernize-use-override, - readability-avoid-const-params-in-decls, - readability-container-size-empty, - readability-redundant-* + portability-simd-intrinsics WarningsAsErrors: '*' HeaderFilterRegex: '.*/(include|src)/.*' FormatStyle: file +CheckOptions: + # `.value()` is a checked access: it throws std::bad_optional_access. + bugprone-unchecked-optional-access.IgnoreValueCalls: true + # A cast to void is an explicit discard (unique_ptr::release once a capsule owns it). + bugprone-unused-return-value.AllowCastToVoid: true + # A shift or mask by a positive literal (`std::size_t{64} << 20`) is not a sign bug. + bugprone-signed-bitwise.IgnorePositiveIntegerLiterals: true + # De Morgan's rewrite of `!(a <= x && x <= b)` changes what a NaN gives: the guards + # are written to refuse NaN. + readability-simplify-boolean-expr.SimplifyDeMorgan: false + # Reference-counted handles, passed by value by convention: Kokkos views (and the + # aliases named ...View), and nanobind's handles and ndarrays (nb::dict, nb::object, + # nb::handle, nb::ndarray and the bindings' ...Input and ...Array aliases of it). + performance-unnecessary-value-param.AllowedTypes: 'View;nb::;nanobind::;Input$;Array$' diff --git a/native/AGENTS.md b/native/AGENTS.md index faa5a0f56..7d3117dfa 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -83,6 +83,8 @@ The native gate runs from the repository root: ```console uv run poe check-native # native/scripts/check.sh uv run poe check-all # solprop's gate (poe check), then the native gate +uv run poe fmt-native # clang-format every tracked C++ file in place +uv run poe tidy-native # clang-tidy the C++ files changed from main (or those named) ``` `check-native` covers this package's Python format, lint and strict typing, C++ format, @@ -100,8 +102,16 @@ wheel on Linux, macOS and Windows, cached under a hash of `native/`, and solprop default lane runs on it on macOS and Windows with the native backend selected, and in the coverage lane on Linux. The `device-compile` lanes compile the CUDA and HIP targets, tests included, on a change to `native/`, with no device. Running on one stays by hand -(`.github/workflows/cuda.yml` on a self-hosted runner, or `pytest -m gpu` on a machine -with a device). +(`.github/workflows/cuda.yml` on a self-hosted runner, with compute-sanitizer's four +tools, or `pytest -m gpu` on a machine with a device). `.github/workflows/native-deep.yml` +runs weekly and reports without gating: source coverage over the native tests, the +extension under ASan and UBSan through the native pytest suite, and +`misc-include-cleaner`. The tools adopted, scheduled and rejected are in +`docs/dev/tooling.md`, "Native C++ tooling". + +Formatting and clang-tidy are pinned in the `dev` group (LLVM 23.1); `.clang-tidy` +enables whole check families with each exclusion stated. A `NOLINT` names its check +and gives its reason on the same line. New behaviour needs a test; a bug fix needs a regression test. Never disable a test, widen a tolerance after seeing a failure, lower warnings or hide a diagnostic to pass @@ -133,8 +143,8 @@ the gate. contracts (lossless copies, integer accounting, identity metadata, hard-mask zeros, explicit no-ops) stay exact. - Kernels may reorder, vectorise, thread and contract. Contraction is on by default - (`-ffp-contract=fast` for GCC, `-ffp-contract=fast-honor-pragmas` for Clang and - `icpx`: plain `fast` makes Clang ignore the pragmas). A protected block (a stable + (`-ffp-contract=fast` for GCC, `-ffp-contract=fast-honor-pragmas` for Clang: plain + `fast` makes Clang ignore the pragmas). A protected block (a stable increment, a small-argument function, a defect or a compensated sum) stays strict with `SOLPROP_NATIVE_STRICT_FUNCTION` and `SOLPROP_NATIVE_STRICT_BLOCK` (`include/solprop_native/fp.hpp`; GCC ignores `#pragma STDC FP_CONTRACT`) and spells diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 7d5fd206c..8bb38be9b 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -78,8 +78,8 @@ elseif(SOLPROP_NATIVE_ENABLE_CUDA AND "The NVIDIA validation build requires GNU C++; " "found ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}.") elseif(NOT SOLPROP_NATIVE_ENABLE_HIP AND - NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|IntelLLVM|Clang|AMDClang)$") - message(FATAL_ERROR "Linux CPU builds require GNU C++, IntelLLVM, or Clang/AOCC.") + NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AMDClang)$") + message(FATAL_ERROR "Linux CPU builds require GNU C++ or Clang/AOCC.") elseif(SOLPROP_NATIVE_ENABLE_CUDA AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) # The CPU reference compiler below, for the host pass of nvcc_wrapper too: GCC 13 @@ -115,48 +115,17 @@ if(NOT APPLE AND NOT SOLPROP_NATIVE_ENABLE_CUDA AND add_compile_options(-fno-fast-math) endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 2022) - message(FATAL_ERROR "Kokkos CPU builds require IntelLLVM 2022 or newer.") - endif() - # icpx defaults to fast floating-point transformations. The adaptive error - # estimate requires precise arithmetic in project and dependency templates. - add_compile_options(-fp-model=precise) - # CMake can find the distribution's older LLVM archiver even when icpx is - # newer. LTO objects require the archive tools shipped with that compiler. - foreach(_tool IN ITEMS ar ranlib) - execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-print-prog-name=llvm-${_tool}" - OUTPUT_VARIABLE _intel_tool OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND_ERROR_IS_FATAL ANY) - if(NOT EXISTS "${_intel_tool}") - message(FATAL_ERROR "icpx could not locate its llvm-${_tool}: ${_intel_tool}") - endif() - string(TOUPPER "${_tool}" _tool_upper) - set(CMAKE_CXX_COMPILER_${_tool_upper} "${_intel_tool}" CACHE FILEPATH - "Archive tool matching IntelLLVM bitcode" FORCE) - set(CMAKE_CXX_COMPILER_${_tool_upper} "${_intel_tool}") - endforeach() - # CMake 3.28 expands these paths during project(), before the correction above. - set(CMAKE_CXX_ARCHIVE_CREATE_IPO - "\"${CMAKE_CXX_COMPILER_AR}\" cr ") - set(CMAKE_CXX_ARCHIVE_APPEND_IPO - "\"${CMAKE_CXX_COMPILER_AR}\" r ") - set(CMAKE_CXX_ARCHIVE_FINISH_IPO - "\"${CMAKE_CXX_COMPILER_RANLIB}\" ") -endif() - # solprop SEP 25: floating-point contraction (fused multiply-add) is on by default for # CPU builds, and the few protected blocks opt out (include/solprop_native/fp.hpp). # Clang's plain `-ffp-contract=fast` ignores `#pragma clang fp contract(off)` and # `#pragma STDC FP_CONTRACT OFF`; `fast-honor-pragmas` contracts across statements and # keeps the protected blocks strict. GCC ignores both pragmas: its protected functions -# carry `optimize("fp-contract=off")` instead. icpx is Clang-based and takes the Clang -# form after `-fp-model=precise`, which alone contracts within a statement only. The -# flag is added after `-fno-fast-math` and `-fp-model=precise`, so neither resets it. +# carry `optimize("fp-contract=off")` instead. The flag is added after +# `-fno-fast-math`, so that does not reset it. # Device builds keep their compilers' defaults (nvcc and HIP Clang already contract). set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "") if(NOT SOLPROP_NATIVE_ENABLE_CUDA AND NOT SOLPROP_NATIVE_ENABLE_HIP) - if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang|IntelLLVM)$") + if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang)$") set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "-ffp-contract=fast-honor-pragmas") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "-ffp-contract=fast") @@ -164,11 +133,6 @@ if(NOT SOLPROP_NATIVE_ENABLE_CUDA AND NOT SOLPROP_NATIVE_ENABLE_HIP) endif() if(SOLPROP_NATIVE_FP_CONTRACT_FLAG) add_compile_options("${SOLPROP_NATIVE_FP_CONTRACT_FLAG}") - # icpx reports the deliberate override of `-fp-model=precise`'s contraction as a - # warning, which `-Werror` would turn into a failed build. - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - add_compile_options(-Wno-overriding-option) - endif() endif() if(WIN32) @@ -249,16 +213,6 @@ endif() # SOLPROP_NATIVE_*_THREADS pools, all read once at import; OMP_NUM_THREADS does not # set these counts. option(SOLPROP_NATIVE_ENABLE_OPENMP "Use OpenMP in Kokkos and FFTW" ON) -# Kokkos Threads (C++ threads) is the host space of a build that must add no OpenMP -# runtime to the process: a fallback for Windows builds (solprop SEP 29). FFTW then uses its own -# threads library, Win32 threads on Windows and POSIX threads elsewhere. It replaces -# OpenMP, so it needs SOLPROP_NATIVE_ENABLE_OPENMP=OFF. -option(SOLPROP_NATIVE_ENABLE_THREADS "Use Kokkos Threads and FFTW's threads library" OFF) -if(SOLPROP_NATIVE_ENABLE_THREADS AND SOLPROP_NATIVE_ENABLE_OPENMP) - message(FATAL_ERROR - "SOLPROP_NATIVE_ENABLE_THREADS replaces OpenMP; also set " - "SOLPROP_NATIVE_ENABLE_OPENMP=OFF.") -endif() # Three build profiles. LOCAL (the default) is the fast build for the machine it is # built on: a vendor BLAS, that machine's own architecture and checked IPO. # DISTRIBUTION is the build for wheels that run on other machines: a documented CPU @@ -289,12 +243,6 @@ option(SOLPROP_NATIVE_ENABLE_CLANG_TIDY "Run clang-tidy while compiling project-owned C++ sources" OFF) option(SOLPROP_NATIVE_ENABLE_SANITIZERS "Instrument project targets with AddressSanitizer and UBSan" OFF) -option(SOLPROP_NATIVE_BUILD_BENCHMARKS - "Build local native microbenchmarks (never installed)" OFF) -set(SOLPROP_NATIVE_GEMM_PROVIDER "HYBRID" CACHE STRING - "Dense transform provider: CUSTOM, KOKKOSBLAS_NATIVE, KOKKOSBLAS_TPL, or HYBRID") -set_property(CACHE SOLPROP_NATIVE_GEMM_PROVIDER PROPERTY STRINGS - CUSTOM KOKKOSBLAS_NATIVE KOKKOSBLAS_TPL HYBRID) set(SOLPROP_NATIVE_ENABLE_TPL_BLAS "AUTO" CACHE STRING "Dispatch compatible GEMMs to vendor BLAS: AUTO (the profile's BLAS), ON or OFF") set_property(CACHE SOLPROP_NATIVE_ENABLE_TPL_BLAS PROPERTY STRINGS AUTO ON OFF) @@ -350,25 +298,11 @@ foreach(SOLPROP_NATIVE_DEVICE_SIZE IN ITEMS endif() endforeach() -string(TOUPPER "${SOLPROP_NATIVE_GEMM_PROVIDER}" - SOLPROP_NATIVE_GEMM_PROVIDER) -set(SOLPROP_NATIVE_GEMM_PROVIDERS - CUSTOM KOKKOSBLAS_NATIVE KOKKOSBLAS_TPL HYBRID) -if(NOT SOLPROP_NATIVE_GEMM_PROVIDER IN_LIST SOLPROP_NATIVE_GEMM_PROVIDERS) - message(FATAL_ERROR - "SOLPROP_NATIVE_GEMM_PROVIDER must be CUSTOM, KOKKOSBLAS_NATIVE, " - "KOKKOSBLAS_TPL, or HYBRID; got ${SOLPROP_NATIVE_GEMM_PROVIDER}.") -endif() -if(SOLPROP_NATIVE_GEMM_PROVIDER STREQUAL "KOKKOSBLAS_TPL" AND - NOT SOLPROP_NATIVE_ENABLE_TPL_BLAS) - message(FATAL_ERROR - "KOKKOSBLAS_TPL requires SOLPROP_NATIVE_ENABLE_TPL_BLAS=ON.") -endif() if(SOLPROP_NATIVE_ENABLE_CUDA AND SOLPROP_NATIVE_ENABLE_HIP) message(FATAL_ERROR "CUDA and HIP are separate build variants and cannot both be enabled.") endif() -if((SOLPROP_NATIVE_ENABLE_OPENMP OR SOLPROP_NATIVE_ENABLE_THREADS) AND +if(SOLPROP_NATIVE_ENABLE_OPENMP AND (SOLPROP_NATIVE_ENABLE_CUDA OR SOLPROP_NATIVE_ENABLE_HIP)) message(FATAL_ERROR "Accelerator variants use a Serial host execution space; disable OpenMP.") @@ -400,11 +334,19 @@ if(SOLPROP_NATIVE_CPU_BASELINE_FLAG) string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${SOLPROP_NATIVE_CPU_BASELINE_FLAG}") endif() string(STRIP "${SOLPROP_NATIVE_COMPILER_FLAGS}" SOLPROP_NATIVE_COMPILER_FLAGS) -string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS - " -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow") -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " -fp-model=precise") +# The project's own sources' warnings, defined once: the targets compile with them and the +# build record lists them. MSVC is not a supported compiler (Windows builds use clang++). +set(SOLPROP_NATIVE_WARNING_FLAGS + -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow -Wundef + -Wnon-virtual-dtor -Woverloaded-virtual) +if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang)$") + list(APPEND SOLPROP_NATIVE_WARNING_FLAGS -Wimplicit-fallthrough) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + list(APPEND SOLPROP_NATIVE_WARNING_FLAGS + -Wduplicated-cond -Wduplicated-branches -Wlogical-op) endif() +list(JOIN SOLPROP_NATIVE_WARNING_FLAGS " " _warning_flags_record) +string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${_warning_flags_record}") if(SOLPROP_NATIVE_FP_CONTRACT_FLAG) string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${SOLPROP_NATIVE_FP_CONTRACT_FLAG}") endif() @@ -422,15 +364,25 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) endif() if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) + # The development group's pinned clang-tidy, in the environment beside the Python + # this build uses, before any machine's LLVM: each LLVM release adds checks. + get_filename_component(_python_bin "${Python_EXECUTABLE}" DIRECTORY) find_program(SOLPROP_NATIVE_CLANG_TIDY clang-tidy - HINTS /opt/homebrew/opt/llvm/bin + HINTS "${_python_bin}" + /opt/homebrew/opt/llvm/bin /usr/local/opt/llvm/bin REQUIRED) + execute_process(COMMAND "${SOLPROP_NATIVE_CLANG_TIDY}" --version + OUTPUT_VARIABLE _clang_tidy_version OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH "version [0-9.]+" _clang_tidy_version "${_clang_tidy_version}") + message(STATUS "clang-tidy: ${SOLPROP_NATIVE_CLANG_TIDY} (${_clang_tidy_version})") set(SOLPROP_NATIVE_CLANG_TIDY_COMMAND "${SOLPROP_NATIVE_CLANG_TIDY}" "--config-file=${PROJECT_SOURCE_DIR}/.clang-tidy" "--header-filter=^${PROJECT_SOURCE_DIR}/(include|src|tests)/.*") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # The GCC-only warnings in the compile commands are unknown to clang-tidy's parser. + list(APPEND SOLPROP_NATIVE_CLANG_TIDY_COMMAND "--extra-arg=-Wno-unknown-warning-option") execute_process( COMMAND "${CMAKE_CXX_COMPILER}" -print-file-name=include OUTPUT_VARIABLE SOLPROP_NATIVE_GNU_INCLUDE_DIR @@ -453,6 +405,20 @@ if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) "--extra-arg=-D__malloc__(...)=__malloc__") endif() endif() + if(APPLE) + # Homebrew's Clang finds the macOS SDK from its own configuration file, which the + # PyPI clang-tidy does not read. + set(_tidy_sysroot "${CMAKE_OSX_SYSROOT}") + if(NOT _tidy_sysroot) + execute_process(COMMAND xcrun --show-sdk-path OUTPUT_VARIABLE _tidy_sysroot + OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) + endif() + list(APPEND SOLPROP_NATIVE_CLANG_TIDY_COMMAND "--extra-arg=-isysroot${_tidy_sysroot}") + endif() + # The command the build runs, the resolved binary first, one argument per line, for + # native/scripts/tidy.sh (`poe tidy-native`) and the weekly include-cleaner report. + list(JOIN SOLPROP_NATIVE_CLANG_TIDY_COMMAND "\n" _tidy_command) + file(WRITE "${PROJECT_BINARY_DIR}/clang_tidy_command.txt" "${_tidy_command}\n") endif() function(solprop_native_configure_project_target target) @@ -462,13 +428,8 @@ function(solprop_native_configure_project_target target) target_link_options(${target} PRIVATE -fprofile-instr-generate) endif() target_compile_options(${target} PRIVATE - $<$:-Wall> - $<$:-Wextra> - $<$:-Wpedantic> - $<$:-Wconversion> - $<$:-Wsign-conversion> - $<$:-Wshadow> - $<$,$>:-Werror> + "$<$:${SOLPROP_NATIVE_WARNING_FLAGS}>" + $<$,$>:-Werror> ) if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) set_property(TARGET ${target} PROPERTY CXX_CLANG_TIDY @@ -476,11 +437,13 @@ function(solprop_native_configure_project_target target) endif() if(SOLPROP_NATIVE_ENABLE_SANITIZERS) target_compile_options(${target} PRIVATE - $<$:-fsanitize=address,undefined> - $<$:-fno-omit-frame-pointer> + $<$:-fsanitize=address,undefined> + # A report stops the test there, whatever UBSAN_OPTIONS says. + $<$:-fno-sanitize-recover=all> + $<$:-fno-omit-frame-pointer> ) target_link_options(${target} PRIVATE - $<$:-fsanitize=address,undefined> + $<$:-fsanitize=address,undefined> ) endif() endfunction() @@ -582,9 +545,6 @@ endif() if(SOLPROP_NATIVE_ENABLE_OPENMP) set(Kokkos_ENABLE_SERIAL OFF CACHE BOOL "Disable the Serial backend" FORCE) set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "Enable the OpenMP backend" FORCE) -elseif(SOLPROP_NATIVE_ENABLE_THREADS) - set(Kokkos_ENABLE_SERIAL OFF CACHE BOOL "Disable the Serial backend" FORCE) - set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Disable the OpenMP backend" FORCE) else() set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Enable the Serial backend" FORCE) set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Disable the OpenMP backend" FORCE) @@ -594,8 +554,7 @@ set(Kokkos_ARCH_NATIVE ${SOLPROP_NATIVE_HOST_ARCH} CACHE BOOL set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ${SOLPROP_NATIVE_ENABLE_AGGRESSIVE_VECTORIZATION} CACHE BOOL "Enable aggressive vectorization of independent Kokkos loops" FORCE) -set(Kokkos_ENABLE_THREADS ${SOLPROP_NATIVE_ENABLE_THREADS} CACHE BOOL - "Enable the Threads backend" FORCE) +set(Kokkos_ENABLE_THREADS OFF CACHE BOOL "OpenMP is the CPU host space" FORCE) set(Kokkos_ENABLE_CUDA ${SOLPROP_NATIVE_ENABLE_CUDA} CACHE BOOL "Enable the CUDA backend" FORCE) set(Kokkos_ENABLE_HIP ${SOLPROP_NATIVE_ENABLE_HIP} CACHE BOOL @@ -760,7 +719,6 @@ endif() message(STATUS "solprop-native build profile: ${SOLPROP_NATIVE_BUILD_PROFILE}; " "BLAS ${SOLPROP_NATIVE_BLAS_BACKEND} (${SOLPROP_NATIVE_BLAS_SELECTION}); " - "GEMM provider ${SOLPROP_NATIVE_GEMM_PROVIDER}; " "native architecture ${SOLPROP_NATIVE_HOST_ARCH}; " "CPU baseline ${SOLPROP_NATIVE_CPU_BASELINE}; " "IPO ${SOLPROP_NATIVE_IPO_AVAILABLE} (Release builds)") @@ -841,7 +799,7 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) target_include_directories(solprop_native_labels SYSTEM PRIVATE "${kokkos_SOURCE_DIR}/core/src") target_compile_options(solprop_native_labels PRIVATE - -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Werror) + ${SOLPROP_NATIVE_WARNING_FLAGS} -Werror) set_target_properties(solprop_native_labels PROPERTIES PREFIX "" CXX_VISIBILITY_PRESET default) install(TARGETS solprop_native_labels LIBRARY DESTINATION solprop_native) @@ -859,7 +817,7 @@ add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp src/general # judgement round as solprop's Python does, so their # translation units take no contraction on any build (the flag follows the global one, # and the last wins). MSVC's /fp:precise does not contract by default. -if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang|IntelLLVM)$") +if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang)$") set_source_files_properties(src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp @@ -876,7 +834,6 @@ target_link_libraries(solprop_native_core PUBLIC ${SOLPROP_NATIVE_MKL_FFT_LIBRARIES} ) target_compile_definitions(solprop_native_core PUBLIC - SOLPROP_NATIVE_GEMM_PROVIDER_${SOLPROP_NATIVE_GEMM_PROVIDER}=1 SOLPROP_NATIVE_DEVICE_TEAM_SIZE=${SOLPROP_NATIVE_DEVICE_TEAM_SIZE} SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE=${SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE} SOLPROP_NATIVE_BAND_STAGING=$ @@ -952,7 +909,6 @@ target_compile_definitions(_core PRIVATE SOLPROP_NATIVE_KOKKOSFFT_VERSION="2.0.0" SOLPROP_NATIVE_KOKKOSKERNELS_VERSION="5.2.0" SOLPROP_NATIVE_SLEEF_VERSION="${SOLPROP_NATIVE_SLEEF_VERSION}" - SOLPROP_NATIVE_GEMM_PROVIDER="${SOLPROP_NATIVE_GEMM_PROVIDER}" SOLPROP_NATIVE_TPL_BLAS_ENABLED=$ SOLPROP_NATIVE_BLAS_BACKEND="${SOLPROP_NATIVE_BLAS_BACKEND}" SOLPROP_NATIVE_BLAS_THREAD_POLICY="${SOLPROP_NATIVE_BLAS_THREAD_POLICY_ENCODED}" @@ -978,13 +934,6 @@ target_compile_definitions(_core PRIVATE SOLPROP_NATIVE_IPO_ENABLED=$,$> ) -if(SOLPROP_NATIVE_BUILD_BENCHMARKS) - add_executable(solprop_native_gemm_benchmark src/benchmark_gemm.cpp) - target_link_libraries(solprop_native_gemm_benchmark PRIVATE solprop_native_core) - solprop_native_mark_dependency_headers_system(solprop_native_gemm_benchmark) - solprop_native_configure_project_target(solprop_native_gemm_benchmark) -endif() - if(BUILD_TESTING) set(INSTALL_GTEST OFF CACHE BOOL "Do not install the test-only GoogleTest" FORCE) set(gtest_force_shared_crt ON CACHE BOOL @@ -1100,7 +1049,9 @@ if(SKBUILD) "openmp_libraries;${OpenMP_CXX_LIB_NAMES}" "fft_backend;${SOLPROP_NATIVE_HOST_FFT_BACKEND}" "blas_backend;${SOLPROP_NATIVE_BLAS_BACKEND}" - "gemm_provider;${SOLPROP_NATIVE_GEMM_PROVIDER}" + # One GEMM provider remains (the vendor GEMM for an eligible transform, the + # custom kernels for the rest); the field stays for the records that hold it. + "gemm_provider;HYBRID" "raman_fft_provider;${SOLPROP_NATIVE_RAMAN_FFT_PROVIDER}" "vector_math;${SOLPROP_NATIVE_VECTOR_MATH}" "build_profile;${SOLPROP_NATIVE_BUILD_PROFILE}" diff --git a/native/CMakePresets.json b/native/CMakePresets.json index 0bfd70245..c93b5a5c0 100644 --- a/native/CMakePresets.json +++ b/native/CMakePresets.json @@ -35,19 +35,19 @@ "inherits": "cpu-debug", "binaryDir": "${sourceDir}/build/presets/cpu-sanitize", "cacheVariables": { - "SOLPROP_NATIVE_ENABLE_SANITIZERS": "ON" + "SOLPROP_NATIVE_ENABLE_SANITIZERS": "ON", + "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON" } }, { "name": "cpu-benchmark", - "displayName": "Local (vendor BLAS, machine-native) CPU benchmark build", + "displayName": "Local (vendor BLAS, machine-native) CPU Release build", "inherits": "cpu-debug", "binaryDir": "${sourceDir}/build/presets/cpu-benchmark", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "SOLPROP_NATIVE_ENABLE_IPO": "ON", - "SOLPROP_NATIVE_PROFILE": "LOCAL", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON" + "SOLPROP_NATIVE_PROFILE": "LOCAL" } }, { @@ -57,7 +57,6 @@ "cacheVariables": { "BUILD_TESTING": "ON", "CMAKE_BUILD_TYPE": "Release", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON", "SOLPROP_NATIVE_ENABLE_CUDA": "ON", "SOLPROP_NATIVE_ENABLE_HIP": "OFF", "SOLPROP_NATIVE_ENABLE_OPENMP": "OFF", @@ -67,43 +66,12 @@ "SOLPROP_NATIVE_HOST_ARCH": "OFF" } }, - { - "name": "cuda-custom", - "displayName": "NVIDIA CUDA custom-GEMM validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-custom", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "CUSTOM", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "OFF" - } - }, - { - "name": "cuda-native", - "displayName": "NVIDIA CUDA native KokkosBlas validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-native", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "KOKKOSBLAS_NATIVE", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "OFF" - } - }, - { - "name": "cuda-tpl", - "displayName": "NVIDIA CUDA cuBLAS validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-tpl", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "KOKKOSBLAS_TPL", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON" - } - }, { "name": "cuda", "displayName": "NVIDIA CUDA hybrid production build", "inherits": "cuda-base", "binaryDir": "${sourceDir}/build/presets/cuda", "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "HYBRID", "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON" } }, @@ -115,12 +83,10 @@ "cacheVariables": { "BUILD_TESTING": "ON", "CMAKE_BUILD_TYPE": "Release", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON", "SOLPROP_NATIVE_ENABLE_CUDA": "OFF", "SOLPROP_NATIVE_ENABLE_HIP": "ON", "SOLPROP_NATIVE_ENABLE_OPENMP": "OFF", "SOLPROP_NATIVE_HIP_ARCH": "AMD_GFX942", - "SOLPROP_NATIVE_GEMM_PROVIDER": "HYBRID", "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON", "SOLPROP_NATIVE_ENABLE_IPO": "OFF", "SOLPROP_NATIVE_HOST_ARCH": "OFF" @@ -132,9 +98,6 @@ {"name": "cpu-tidy", "configurePreset": "cpu-tidy"}, {"name": "cpu-sanitize", "configurePreset": "cpu-sanitize"}, {"name": "cpu-benchmark", "configurePreset": "cpu-benchmark"}, - {"name": "cuda-custom", "configurePreset": "cuda-custom"}, - {"name": "cuda-native", "configurePreset": "cuda-native"}, - {"name": "cuda-tpl", "configurePreset": "cuda-tpl"}, {"name": "cuda", "configurePreset": "cuda"}, {"name": "hip", "configurePreset": "hip"} ] diff --git a/native/README.md b/native/README.md index ab16d0dde..c534e4fe6 100644 --- a/native/README.md +++ b/native/README.md @@ -91,15 +91,21 @@ and `tests/test_band_stress.py` with `SOLPROP_NATIVE_BAND_TILES=4`; on a GPU, ```sh uv run poe check-native # at the repository root; runs native/scripts/check.sh uv run poe check-all # solprop's gate, then this one +uv run poe fmt-native # format every tracked C++ file in place +uv run poe tidy-native # clang-tidy on the C++ files changed from main, or on those named ``` `poe check-native` runs everything a change to `native/` must pass: Ruff formatting and -lint, strict Pyright, the pinned `clang-format` from the `dev` group, clang-tidy with -strict warnings, the native tests, the native tests under ASan and UBSan, an editable +lint, strict Pyright, the pinned `clang-format` from the `dev` group over every tracked +C++ file, the pinned clang-tidy (whole check families, `.clang-tidy`) with strict +warnings, the native tests, the native tests under ASan and UBSan, an editable local build with the Python tests, a distribution wheel build, and an import smoke test of that wheel in a clean environment (on Linux it checks that the wheel's OpenBLAS loads from the package). It syncs the repository's environment without removing the native -build, honours `CMAKE_ARGS`, and needs Node.js (`npx`) for Pyright. +build, honours `CMAKE_ARGS`, and needs Node.js (`npx`) for Pyright. `SOLPROP_NATIVE_CHECK_STAGES` +picks stages (`lint`, `tidy`, `sanitize`, `pytest`, `wheel`). The slower checks run +weekly in `.github/workflows/native-deep.yml` and report without gating: source +coverage, the extension under ASan and UBSan, and an include-cleaner report. `SOLPROP_NATIVE_QUALIFICATION_PYTHON=/path/to/python` installs the wheel into that environment and runs the Python tests there instead. A change to the backend also passes solprop's own suite with this package installed (`poe check-all`, or `uv run poe test` diff --git a/native/benchmarks/cpu_timer.cpp b/native/benchmarks/cpu_timer.cpp deleted file mode 100644 index 3a459366c..000000000 --- a/native/benchmarks/cpu_timer.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -// CPU-only attribution tool for the pinned Kokkos profiling callback ABI. -// OpenMP kernels finish synchronously; disabling tool fences leaves their order intact. -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { -using Clock = std::chrono::steady_clock; -struct Event { - std::string name; - Clock::time_point start; -}; -struct Total { - std::uint64_t calls{}; - double seconds{}; -}; -bool enabled{}; -std::vector events; -std::vector regions; -std::map totals; -void begin(const char* name, std::uint64_t* id) { - *id = 0; - if (enabled) { - events.push_back({name, Clock::now()}); - *id = events.size(); - } -} -void end(std::uint64_t id) { - if (id != 0 && enabled) { - const auto& event = events.at(id - 1); - auto& total = totals[event.name]; - total.seconds += std::chrono::duration(Clock::now() - event.start).count(); - ++total.calls; - } -} -} // namespace -extern "C" { -void kokkosp_request_tool_settings(std::uint32_t, Kokkos_Tools_ToolSettings* settings) { - settings->requires_global_fencing = false; -} -void kokkosp_begin_parallel_for(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_for(std::uint64_t id) { end(id); } -void kokkosp_begin_parallel_reduce(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_reduce(std::uint64_t id) { end(id); } -void kokkosp_begin_parallel_scan(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_scan(std::uint64_t id) { end(id); } -void kokkosp_push_profile_region(const char* name) { - std::uint64_t id{}; - begin(name, &id); - regions.push_back(id); -} -void kokkosp_pop_profile_region() { - end(regions.back()); - regions.pop_back(); -} -void solprop_timer_start() { - events.clear(); - totals.clear(); - enabled = true; -} -void solprop_timer_stop(const char* path) { - enabled = false; - std::ofstream output(path); - output.precision(17); - output << "name\tcalls\tseconds\n"; - for (const auto& [name, total] : totals) - output << name << '\t' << total.calls << '\t' << total.seconds << '\n'; -} -} diff --git a/native/benchmarks/ctc.py b/native/benchmarks/ctc.py deleted file mode 100644 index c173f81ce..000000000 --- a/native/benchmarks/ctc.py +++ /dev/null @@ -1,552 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure the reduced and full-shape CtC cases in isolated processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances, each plane's -statistics evaluated there, up to the last plane's observation, before its field is -exported. A trial is the backend's timing probe (``Backend.timing_probe``), which loads -the start once and times single trials from it. ``--planes`` keeps a run's first planes -only, for a short check of the script. The untimed warm-up is the launch plane and one -attempt of a session of the timed run itself, closed there: a backend serves only its -own request's planes, where the step driver's warm-up marched to a short plane of its -own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import json -import math -import os -import statistics -import subprocess -import sys -import threading -import time -import warnings -from typing import Any, TypedDict, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One isolated CtC measurement.""" - - backend: str - device_before: DeviceRecord - case: str - mode: str - seconds: float - construction_seconds: float - planning_seconds: float - raman_planning_seconds: float - trials: int - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - modes: int - nodes: int - node_chunk: int - components: int - n_omega: int - n_to: int - raman_terms: int - raman_kernel_transforms: int - raman_memory_ledger: dict[str, int] - field_sha256: str - memory_ledger_bytes: int - kernel_threads: int - native_threads: int - fftw_threads: int - blas_threads: int | str - gemm_provider: str - synthesis_provider: str - projection_provider: str - blas_backend: str - propagator_mode: str - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus spread across isolated processes.""" - - minimum_seconds: float - maximum_seconds: float - median_absolute_deviation: float - - -# The two CtC shapes, defined here rather than read from a stored comparison: hydrogen -# at 40 bar in a 16 um capillary, 400 fs and 10 uJ at 515 nm, no loss, the package's -# own materials. The full shape runs 0.3 m on six modes and a 4 ps, 170 nm-4 um grid; -# the reduced one is the fundamental alone over 0.1 m on a 2 ps, 250 nm-2 um grid, five -# times the pulse, which keeps the launch clear of the temporal collar. -class Shape(TypedDict): - """One CtC shape: its length, modes, band, window and save count.""" - - length: float - modes: int - wavelength_limits: tuple[float, float] - trange: float - n_save: int - - -CASES: dict[str, Shape] = { - "ctc_reduced": { - "length": 0.1, - "modes": 1, - "wavelength_limits": (250e-9, 2e-6), - "trange": 2e-12, - "n_save": 21, - }, - "ctc_full": { - "length": 0.3, - "modes": 6, - "wavelength_limits": (170e-9, 4e-6), - "trange": 4e-12, - "n_save": 201, - }, -} - - -def build_simulation(case_name: str, backend_name: str) -> Any: - from solprop.driver import Simulation - - case = CASES[case_name] - return Simulation.from_capillary( - 16e-6, - case["length"], - "H2", - 40.0, - wavelength=515e-9, - wavelength_limits=case["wavelength_limits"], - trange=case["trange"], - fwhm=400e-15, - energy=10e-6, - modes=case["modes"], - loss=False, - raman=True, - n_save=case["n_save"], - rtol=1e-6, - atol=1e-10, - backend=backend_name, - ) - - -def _metadata_int(metadata: dict[str, object], name: str, default: int) -> int: - value = metadata.get(name, default) - if isinstance(value, bool) or not isinstance(value, int | str): - raise TypeError(f"metadata {name!r} is not integer-valued: {value!r}") - return int(value) - - -def _metadata_float(metadata: dict[str, object], name: str) -> float: - value = metadata.get(name, 0.0) - if isinstance(value, bool) or not isinstance(value, int | float): - raise TypeError(f"metadata {name!r} is not numeric: {value!r}") - return float(value) - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _worker( - backend_name: str, - case_name: str, - mode: str, - trial_count: int, - node_chunk: int | None, - progress_seconds: float, - planes_kept: int | None, -) -> None: - if backend_name == "native": - __import__("solprop_native") - - import numpy as np - - from solprop.backends.node_chunk import plan_node_chunk - from solprop.backends.registry import select - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - from solprop.backends.session import ExecutionOptions, Start - from solprop.backends.sessions import start_session - from solprop.exceptions import MaterialRangeWarning - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", MaterialRangeWarning) - simulation = build_simulation(case_name, backend_name) - problem = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - if planes_kept is not None: - request = request.with_stepper( - dataclasses.replace( - request.stepper, save_planes=request.stepper.save_planes[:planes_kept] - ) - ) - options = None if node_chunk is None else ExecutionOptions(node_chunk=node_chunk) - - constructed = time.perf_counter() - backend: Any = select( - backend_name, - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - options=options, - ) - node_chunk_used = ( - int(backend.node_chunk) - if backend_name == "native" - else plan_node_chunk(problem, options).chunk - ) - construction_seconds = time.perf_counter() - constructed - dz = float(request.stepper.dz_init) - identity = ExportRecipe(power=False, optics=False) - - def entrance() -> Start: - return Start.entrance(problem, noise=simulation.noise) - - try: - if mode == "trial": - probe = backend.timing_probe(entrance()) - device_before = device_record(backend.metadata()) - probe.trial(0.0, dz) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - load_end = os.getloadavg()[0] - probe.trial(0.0, dz) - field = probe.field() - accepted = 0 - rejected = 0 - trials = trial_count - else: - device_before = device_record(backend.metadata()) - _warm(start_session(backend, problem, request, entrance(), owns=False)) - session = start_session(backend, problem, request, entrance(), owns=False) - load_start = os.getloadavg()[0] - started = time.perf_counter() - process_started = time.process_time() - attempts = 0 - rejected = 0 - save_planes = [float(plane) for plane in request.stepper.save_planes] - planes = len(save_planes) - progress_state: dict[str, int | float] = { - "completed_planes": 0, - "current_plane": 1, - "target": save_planes[0], - "accepted_steps": 0, - "rejected_steps": 0, - } - progress_stop = threading.Event() - - def emit_progress(kind: str) -> None: - record: dict[str, bool | float | int | str] = { - "progress": True, - "kind": kind, - "planes": planes, - "elapsed_seconds": time.perf_counter() - started, - "process_cpu_seconds": time.process_time() - process_started, - **progress_state, - } - print(json.dumps(record, sort_keys=True), file=sys.stderr, flush=True) - - def progress_heartbeat() -> None: - while not progress_stop.wait(progress_seconds): - emit_progress("heartbeat") - - progress_thread: threading.Thread | None = None - if progress_seconds > 0.0: - progress_thread = threading.Thread( - target=progress_heartbeat, - name="ctc-progress", - daemon=True, - ) - progress_thread.start() - seconds = 0.0 - field = None - try: - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += history.n_steps - rejected += history.n_rejected - if not isinstance(event, ObservationReady): - continue - if event.plane_index < 0: - continue # the launch plane - plane = event.plane_index + 1 - progress_state["completed_planes"] = plane - progress_state["accepted_steps"] = attempts - rejected - progress_state["rejected_steps"] = rejected - if plane < planes: - progress_state["current_plane"] = plane + 1 - progress_state["target"] = save_planes[plane] - continue - if progress_seconds > 0.0: - emit_progress("complete") - seconds = time.perf_counter() - started - field = np.array(session.export(event.ticket, identity)) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - progress_stop.set() - if progress_thread is not None: - progress_thread.join() - session.close() - load_end = os.getloadavg()[0] - accepted = attempts - rejected - trials = attempts - if field is None: - raise RuntimeError("the CtC session observed no last plane") - if not np.all(np.isfinite(field)): - raise RuntimeError("CtC benchmark produced a non-finite field") - metadata: dict[str, object] = backend.metadata() - device_after = device_record(metadata) - ledger = backend.memory_ledger() - raman_transforms = int(getattr(backend, "_raman_transforms", 0)) - finally: - backend.close() - - threads = _metadata_int(metadata, "thread_count", 0) - fftw_threads = _metadata_int(metadata, "fftw_threads", threads) - blas_backend = str(metadata.get("blas_backend", "not applicable")) - if blas_backend == "Accelerate": - blas_threads: int | str = "system-managed" - elif blas_backend == "OpenBLAS": - blas_threads = int(os.environ.get("OPENBLAS_NUM_THREADS", "1")) - elif blas_backend in {"cuBLAS", "rocBLAS"}: - blas_threads = "device-stream" - elif backend_name == "native": - blas_threads = "disabled" - else: - blas_threads = int(os.environ.get("SOLPROP_BLAS_THREADS", "1")) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "case": case_name, - "mode": mode, - "seconds": seconds, - "construction_seconds": construction_seconds, - "planning_seconds": _metadata_float(metadata, "planning_seconds"), - "raman_planning_seconds": _metadata_float(metadata, "raman_planning_seconds"), - "trials": trials, - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "modes": problem.shape[0], - "nodes": problem.transform.n_nodes, - "node_chunk": node_chunk_used, - "components": problem.transform.n_pol * problem.shape[1], - "n_omega": problem.shape[-1], - "n_to": problem.grids.n_to, - "raman_terms": ( - 0 - if problem.responses.raman_coefficients is None - else problem.responses.raman_coefficients.n_terms - ), - "raman_kernel_transforms": raman_transforms, - "raman_memory_ledger": { - name: ledger[name] - for name in ( - "node_raman_drive", - "node_raman_spectrum", - "node_raman_convolved", - "raman_undamped", - "raman_kernel_time", - "raman_kernel_spectrum", - "stage_raman_rates", - ) - if name in ledger - }, - "field_sha256": hashlib.sha256(field.tobytes(order="C")).hexdigest(), - "memory_ledger_bytes": sum(ledger.values()), - "kernel_threads": threads, - "native_threads": threads if backend_name == "native" else 0, - "fftw_threads": fftw_threads, - "blas_threads": blas_threads, - "gemm_provider": str(metadata.get("gemm_provider", "not applicable")), - "synthesis_provider": str(metadata.get("synthesis_provider", "not applicable")), - "projection_provider": str( - metadata.get("projection_provider", "not applicable") - ), - "blas_backend": blas_backend, - "propagator_mode": str(metadata.get("propagator_mode", "unknown")), - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _csv(value: str) -> list[str]: - values = [item.strip() for item in value.split(",") if item.strip()] - if not values: - raise argparse.ArgumentTypeError("the list must not be empty") - return values - - -def _run( - backend: str, - threads: int, - blas_threads: int, - case_name: str, - mode: str, - trial_count: int, - node_chunk: int | None, - progress_seconds: float, - planes_kept: int | None, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "OPENBLAS_NUM_THREADS": str(blas_threads), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - environment.setdefault("OMP_WAIT_POLICY", "active") - command = [ - sys.executable, - __file__, - "--worker", - backend, - "--case", - case_name, - "--mode", - mode, - "--trial-count", - str(trial_count), - ] - if node_chunk is not None: - command.extend(("--node-chunk", str(node_chunk))) - if progress_seconds > 0.0: - command.extend(("--progress-seconds", str(progress_seconds))) - if planes_kept is not None: - command.extend(("--planes", str(planes_kept))) - completed = subprocess.run( - command, - check=True, - stdout=subprocess.PIPE, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run isolated CtC workers and emit JSON samples with robust summaries.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument( - "--case", choices=("ctc_reduced", "ctc_full"), default="ctc_reduced" - ) - parser.add_argument("--mode", choices=("trial", "propagation"), default="trial") - parser.add_argument("--trial-count", type=int, default=5) - parser.add_argument("--node-chunk", type=int) - parser.add_argument("--progress-seconds", type=float, default=0.0) - parser.add_argument("--backends", type=_csv, default=["native"]) - parser.add_argument("--threads", type=_csv, default=["8"]) - parser.add_argument("--blas-threads", type=int, default=1) - parser.add_argument("--repeats", type=int, default=1) - parser.add_argument( - "--planes", - type=int, - help="run the case's first planes only, to check the script", - ) - arguments = parser.parse_args() - if arguments.planes is not None and arguments.planes < 1: - parser.error("--planes must be positive") - if min(arguments.trial_count, arguments.blas_threads, arguments.repeats) < 1: - parser.error("counts must be positive") - if arguments.node_chunk is not None and arguments.node_chunk < 1: - parser.error("--node-chunk must be positive") - if not math.isfinite(arguments.progress_seconds) or arguments.progress_seconds < 0: - parser.error("--progress-seconds must be finite and non-negative") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.case, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - arguments.progress_seconds, - arguments.planes, - ) - return - for backend in arguments.backends: - for thread_text in arguments.threads: - threads = int(thread_text) - samples = [ - _run( - backend, - threads, - arguments.blas_threads, - arguments.case, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - arguments.progress_seconds, - arguments.planes, - ) - for _ in range(arguments.repeats) - ] - median = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - median)) - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - "median_absolute_deviation": statistics.median( - abs(sample["seconds"] - median) for sample in samples - ), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/device_record.py b/native/benchmarks/device_record.py deleted file mode 100644 index 819f1c5eb..000000000 --- a/native/benchmarks/device_record.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Small machine record shared by isolated accelerator benchmarks.""" - -from __future__ import annotations - -import csv -import io -import subprocess -from collections.abc import Mapping -from typing import TypedDict - - -class DeviceRecord(TypedDict): - """Execution-space and live accelerator telemetry for one sample.""" - - device_backend: str - device_arch: str - device_name: str - device_uuid: str - device_memory_total_mib: float - device_memory_used_mib: float - device_utilization_percent: float - device_temperature_c: float - device_pstate: str - device_clock_mhz: float - device_power_w: float - device_team_size: int - device_reduction_size: int - - -def _number(value: str) -> float: - """Parse one no-units NVIDIA CSV value, retaining unavailable values as NaN.""" - try: - return float(value) - except ValueError: - return float("nan") - - -def _integer_metadata(metadata: Mapping[str, object], name: str) -> int: - """Read one integer-valued native metadata field without silent coercion.""" - value = metadata.get(name, 0) - if isinstance(value, bool) or not isinstance(value, (int, str)): - raise TypeError(f"metadata field {name!r} is not an integer") - return int(value) - - -def device_record(metadata: Mapping[str, object]) -> DeviceRecord: - """Return build identity and one post-sample NVIDIA telemetry snapshot.""" - backend = str(metadata.get("device_backend", "none")) - record: DeviceRecord = { - "device_backend": backend, - "device_arch": str(metadata.get("device_arch", "host")), - "device_name": "not applicable", - "device_uuid": "not applicable", - "device_memory_total_mib": 0.0, - "device_memory_used_mib": 0.0, - "device_utilization_percent": 0.0, - "device_temperature_c": 0.0, - "device_pstate": "not applicable", - "device_clock_mhz": 0.0, - "device_power_w": 0.0, - "device_team_size": _integer_metadata(metadata, "device_team_size"), - "device_reduction_size": _integer_metadata(metadata, "device_reduction_size"), - } - if backend != "cuda": - return record - query = ( - "name,uuid,memory.total,memory.used,utilization.gpu,temperature.gpu," - "pstate,clocks.sm,power.draw" - ) - completed = subprocess.run( - [ - "nvidia-smi", - f"--query-gpu={query}", - "--format=csv,noheader,nounits", - ], - check=True, - capture_output=True, - text=True, - ) - rows = list(csv.reader(io.StringIO(completed.stdout), skipinitialspace=True)) - if len(rows) != 1 or len(rows[0]) != 9: - raise RuntimeError("CUDA benchmarks require exactly one visible NVIDIA GPU") - values = rows[0] - record.update( - { - "device_name": values[0], - "device_uuid": values[1], - "device_memory_total_mib": _number(values[2]), - "device_memory_used_mib": _number(values[3]), - "device_utilization_percent": _number(values[4]), - "device_temperature_c": _number(values[5]), - "device_pstate": values[6], - "device_clock_mhz": _number(values[7]), - "device_power_w": _number(values[8]), - } - ) - return record diff --git a/native/benchmarks/gemm_shapes.py b/native/benchmarks/gemm_shapes.py deleted file mode 100644 index aaae91a09..000000000 --- a/native/benchmarks/gemm_shapes.py +++ /dev/null @@ -1,219 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure a dense transform's cost per source call by shape, for one build. - -The `HYBRID` GEMM policy chooses between the vendor GEMM and the custom kernels by -shape. This times ten fixed IP-DOPRI5 steps (sixty-one source calls) of short solprop -problems whose transforms span the shapes the policy decides on -- a radial field -route from 2 to 256 nodes, and one- to six-mode capillaries as real fields and as -envelopes -- and records each problem's chosen providers beside its median cost per -call over five warmed repetitions. Run it once per build to compare policies (for -example the installed build against one with ``SOLPROP_NATIVE_GEMM_PROVIDER`` forced), -and on a device build to find that device's crossover:: - - OMP_NUM_THREADS=1 SOLPROP_NATIVE_THREADS=1 VECLIB_MAXIMUM_THREADS=1 \\ - python benchmarks/gemm_shapes.py --output gemm-shapes.json - -Each repetition is one run of the backend's session (solprop SEP 47), whose start -loads the launch before the clock starts; the timed region is its advances to the -end. ``--nodes`` and ``--modes`` measure a subset of the shapes. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import json -import os -import statistics -import time -import warnings -from pathlib import Path -from typing import Any, TypedDict - -import solprop_native -import numpy as np -from solprop.backends.registry import select -from solprop.backends.session import Start -from solprop.backends.sessions import start_session -from solprop.driver import Simulation, SimulationSettings -from solprop.grid import FieldGrid, RadialGrid -from solprop.nonlinear.responses import KerrResponse -from solprop.physdata import gamma3 -from solprop.pulses import GaussianPulse, SpatialPulse -from solprop.spatial import SpatialMedium -from solprop.stepper import FixedSchedule, PIController - -STEPS = 10 -"""Fixed steps a radial problem takes: six source calls each and one to start.""" - - -class Row(TypedDict): - """One problem's shape, its providers and its cost per source call.""" - - kind: str - size: int - envelope: bool | None - shape: list[int] - n_t: int - synthesis: str - projection: str - calls: int - per_call_ms: float - - -def radial(nodes: int, trange: float) -> Simulation: - """Return a radial field route of `nodes` Hankel nodes on a fixed schedule.""" - length = 1e-3 - grid = FieldGrid.from_limits(length, 800e-9, (500e-9, 1.6e-6), trange) - settings = SimulationSettings( - PIController(rtol=1e-6, atol=0.0), - save_planes=[length], - dz_max=length / STEPS, - dz_init=length / STEPS, - schedule=FixedSchedule(np.full(STEPS, length / STEPS)), - backend="numpy_ref", - ) - pulse = SpatialPulse( - GaussianPulse(wavelength=800e-9, fwhm=30e-15, energy=1e-6), 100e-6 - ) - return Simulation.from_radial( - grid, - RadialGrid(1e-3, nodes), - SpatialMedium("Ar", 1.0), - [KerrResponse(gamma3("Ar", wavelength=800e-9), thg=False)], - pulse, - settings=settings, - ) - - -def capillary(modes: int, trange: float, *, envelope: bool) -> Simulation: - """Return a short argon capillary of `modes` modes, stepped adaptively.""" - length = 1e-3 - return Simulation.from_capillary( - 75e-6, - length, - "Ar", - 1.0, - wavelength=800e-9, - wavelength_limits=(300e-9, 2e-6), - trange=trange, - envelope=envelope, - fwhm=30e-15, - energy=10e-6, - modes=modes, - plasma=False, - raman=False, - rtol=1e-6, - backend="numpy_ref", - save_planes=[length], - ) - - -def measure(simulation: Simulation, kind: str, size: int, envelope: bool | None) -> Row: - """Time one problem's march five times after a warm one; return its row.""" - problem: Any = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - seconds: list[float] = [] - calls = 0 - providers = ("", "") - for repetition in range(6): - backend: Any = select( - "native", - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - if repetition == 0: - native = backend._native_state - providers = ( - str(native.synthesis_provider), - str(native.projection_provider), - ) - session = start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - attempts = 0 - try: - clock = time.perf_counter() - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += int(history.n_steps) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - seconds.append(time.perf_counter() - clock) - finally: - session.close() - backend.close() - calls = 6 * attempts + 1 - return { - "kind": kind, - "size": size, - "envelope": envelope, - "shape": [int(value) for value in problem.shape], - "n_t": int(problem.grids.n_t), - "synthesis": providers[0], - "projection": providers[1], - "calls": calls, - "per_call_ms": 1e3 * statistics.median(seconds[1:]) / calls, - } - - -def _counts(text: str) -> tuple[int, ...]: - """Parse a comma-separated list of positive counts.""" - counts = tuple(int(part) for part in text.split(",") if part.strip()) - if any(count < 1 for count in counts): - raise argparse.ArgumentTypeError("counts must be positive") - return counts - - -def main() -> None: - """Measure every shape for the importable build and write the rows.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument( - "--nodes", - type=_counts, - default=(2, 4, 8, 16, 32, 64, 128, 256), - help="the radial node counts, comma-separated; empty for none", - ) - parser.add_argument( - "--modes", - type=_counts, - default=(1, 2, 3, 6), - help="the capillary mode counts, comma-separated; empty for none", - ) - args = parser.parse_args() - warnings.filterwarnings("ignore") - rows: list[Row] = [] - for nodes in args.nodes: - for trange in (100e-15, 400e-15): - rows.append(measure(radial(nodes, trange), "radial", nodes, None)) - print(rows[-1], flush=True) - for modes in args.modes: - for trange in (200e-15, 800e-15): - for envelope in (False, True): - simulation = capillary(modes, trange, envelope=envelope) - rows.append(measure(simulation, "capillary", modes, envelope)) - print(rows[-1], flush=True) - record = { - "build": dataclasses.asdict(solprop_native.build_info()), - "load": os.getloadavg(), - "rows": rows, - } - args.output.write_text(json.dumps(record, indent=2) + "\n") - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m1.py b/native/benchmarks/m1.py deleted file mode 100644 index ef28479da..000000000 --- a/native/benchmarks/m1.py +++ /dev/null @@ -1,408 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure the full N=6 soliton run in isolated backend/thread processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances to the one save -plane, stopped when that plane's observation arrives, before its field is exported. A -trial is the backend's timing probe (``Backend.timing_probe``), which loads the start -once and times single trials from it. The untimed warm-up is the launch plane and one -attempt of a session of the timed run itself, closed there: a backend serves only its -own request's planes, where the step driver's warm-up marched to a short plane of its -own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import json -import os -import statistics -import subprocess -import sys -import time -from typing import Any, NamedTuple, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One validated M1 measurement.""" - - backend: str - device_before: DeviceRecord - mode: str - trials: int - threads: int - realized_threads: int - fftw_threads: int - seconds: float - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - target_fraction: float - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - kmp_blocktime: str - grid_scale: float - n_omega: int - n_to: int - work_elements: int - warmed: bool - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus the observed three-run interval.""" - - minimum_seconds: float - maximum_seconds: float - - -class Marched(NamedTuple): - """One session's march to its last plane.""" - - seconds: float - accepted: int - rejected: int - field: Any - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _march(session: Any, last: float) -> Marched: - """Advance a started session to its end, timing it up to the plane at `last`. - - The clock stops when that plane's observation arrives, before its field is - exported (the internal convention, as the step driver observed it). - """ - import numpy as np - - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - - accepted = rejected = 0 - seconds = 0.0 - field = None - try: - started = time.perf_counter() - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - taken = int(np.count_nonzero(history.accepted)) - accepted += taken - rejected += history.n_steps - taken - if isinstance(event, ObservationReady) and event.z == last: - seconds = time.perf_counter() - started - field = np.array( - session.export(event.ticket, ExportRecipe(False, False)) - ) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - if field is None: - raise RuntimeError(f"the session observed no plane at {last} m") - return Marched(seconds, accepted, rejected, field) - - -def _worker( - backend_name: str, - target_fraction: float, - grid_scale: float, - warmup: bool, - mode: str, - trial_count: int, -) -> None: - if backend_name == "native": - # macOS must load the extension's OpenMP runtime before any other. - __import__("solprop_native") - - import numpy as np - - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.grid import EnvelopeGrid - from solprop.nonlinear.gnlse import problem_from - from solprop.pulses import SechPulse - from solprop.stepper import PIController - from solprop.translate.problem import assemble - - beta2 = -11.83e-27 - gamma = 0.11 - tau0 = 100e-15 - wavelength = 835e-9 - period = np.pi / 2 * tau0**2 / abs(beta2) - target = target_fraction * period - grid = EnvelopeGrid.from_limits(1.0, wavelength, (500e-9, 2e-6), grid_scale * 4e-12) - model = problem_from( - [0.0, 0.0, beta2], - omega_ref=grid.omega0, - gamma=gamma, - shock=False, - ) - peak_power = 36 * abs(beta2) / (gamma * tau0**2) - field = SechPulse(wavelength=wavelength, width=tau0, peak_power=peak_power).build( - grid - ) - problem, request = assemble( - grid, - model.mode, - [model.kerr], - field, - controller=PIController(rtol=1e-10, atol=1e-18), - save_planes=[target], - dz_max=period, - dz_init=1e-4, - density=1.0, - shock=False, - prefactor_index=1.0, - ) - backend: Any = select(backend_name, problem, request, grid=grid) - dz = float(request.stepper.dz_init) - try: - if mode == "trial": - probe = backend.timing_probe(Start.entrance(problem)) - if warmup: - probe.trial(0.0, dz) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - probe.trial(0.0, dz) - field_out = probe.field() - accepted = 0 - rejected = 0 - trials = trial_count - else: - if warmup: - _warm( - start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - ) - session = start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - marched = _march(session, target) - seconds = marched.seconds - attempts = marched.accepted + marched.rejected - accepted = marched.accepted - rejected = marched.rejected - # Every trial is an attempt, accepted or rejected. (The step driver's - # summary path added the rejections twice: `n_steps` already held them.) - trials = attempts - field_out = marched.field - load_end = os.getloadavg()[0] - if not np.all(np.isfinite(field_out)): - raise RuntimeError("M1 produced a non-finite field") - metadata = backend.metadata() - device_after = device_record(metadata) - finally: - backend.close() - - recorded_threads = metadata.get("thread_count") - realized = ( - recorded_threads - if isinstance(recorded_threads, int) - else int(os.environ["BENCH_THREADS"]) - ) - recorded_fftw_threads = metadata.get("fftw_threads") - realized_fftw_threads = ( - recorded_fftw_threads if isinstance(recorded_fftw_threads, int) else realized - ) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "mode": mode, - "trials": trials, - "threads": int(os.environ["BENCH_THREADS"]), - "realized_threads": realized, - "fftw_threads": realized_fftw_threads, - "seconds": seconds, - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "target_fraction": target_fraction, - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "kmp_blocktime": os.environ.get("KMP_BLOCKTIME", ""), - "grid_scale": grid_scale, - "n_omega": grid.n_omega, - "n_to": grid.n_to, - "work_elements": grid.n_to, - "warmed": warmup, - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _parse_csv(value: str) -> list[str]: - result = [part.strip() for part in value.split(",") if part.strip()] - if not result: - raise argparse.ArgumentTypeError("the list must not be empty") - return result - - -def _run( - backend: str, - threads: int, - target_fraction: float, - grid_scale: float, - warmup: bool, - mode: str, - trial_count: int, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "BENCH_THREADS": str(threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - command = [ - sys.executable, - __file__, - "--worker", - backend, - "--fraction", - str(target_fraction), - "--grid-scale", - str(grid_scale), - "--mode", - mode, - "--trial-count", - str(trial_count), - ] - if warmup: - command.append("--warmup") - completed = subprocess.run( - command, - check=True, - capture_output=True, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run workers and print the median end-to-end M1 table as JSON lines.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument("--backends", type=_parse_csv, default=["numpy_ref", "native"]) - parser.add_argument("--threads", type=_parse_csv, default=["1", "8"]) - parser.add_argument("--repeats", type=int, default=3) - parser.add_argument( - "--mode", choices=("trial", "propagation"), default="propagation" - ) - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument( - "--fraction", - type=float, - default=1.0, - help="fraction of the full soliton period to propagate", - ) - parser.add_argument( - "--warmup", - action="store_true", - help="run one untimed adaptive step before the measured run", - ) - parser.add_argument( - "--grid-scale", - type=float, - default=1.0, - help="multiple of the M1 time window, and therefore approximately its FFT size", - ) - arguments = parser.parse_args() - if not 0.0 < arguments.fraction <= 1.0: - parser.error("--fraction must be in (0, 1]") - if arguments.grid_scale <= 0.0: - parser.error("--grid-scale must be positive") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.fraction, - arguments.grid_scale, - arguments.warmup, - arguments.mode, - arguments.trial_count, - ) - return - if arguments.repeats < 1 or arguments.trial_count < 1: - parser.error("--repeats and --trial-count must be positive") - - for backend in arguments.backends: - if backend not in {"numpy_ref", "native"}: - parser.error(f"unknown backend {backend!r}") - for thread_text in arguments.threads: - threads = int(thread_text) - if threads < 1: - parser.error("thread counts must be positive") - samples = [ - _run( - backend, - threads, - arguments.fraction, - arguments.grid_scale, - arguments.warmup, - arguments.mode, - arguments.trial_count, - ) - for _ in range(arguments.repeats) - ] - middle = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - middle)) - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m2.py b/native/benchmarks/m2.py deleted file mode 100644 index f783475d9..000000000 --- a/native/benchmarks/m2.py +++ /dev/null @@ -1,487 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure reduced M2 propagation in isolated backend/thread processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances to its last -plane, with every observation's field exported (the internal convention, as the step -driver observed it) and its statistics, which the session evaluates there. A trial is -the backend's timing probe (``Backend.timing_probe``), which loads the start once and -times single trials from it. ``--planes`` keeps a run's first planes only, for a short -check of the script. The untimed warm-up is the launch plane and one attempt of a -session of the timed run itself, closed there: a backend serves only its own request's -planes, where the step driver's warm-up marched to a short plane of its own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import json -import os -import statistics -import subprocess -import sys -import time -from collections.abc import Mapping -from pathlib import Path -from typing import Any, TypedDict, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One reduced-M2 propagation measurement.""" - - backend: str - device_before: DeviceRecord - mode: str - trials: int - case: str - field_family: str - seconds: float - construction_seconds: float - planning_seconds: float - memory_ledger: dict[str, int] - memory_ledger_bytes: int - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - modes: int - nodes: int - node_chunk: int - components: int - n_omega: int - n_to: int - warmed: bool - native_threads: int - fftw_threads: int - blas_threads: int | str - blas_thread_policy: str - gemm_provider: str - synthesis_provider: str - projection_provider: str - blas_backend: str - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus its spread across isolated processes.""" - - minimum_seconds: float - maximum_seconds: float - median_absolute_deviation: float - - -class CaseTable(TypedDict): - """Geometry and sampling values of the M2 benchmark cases (``m2_cases.py``).""" - - radius: float - length: float - pressure: float | tuple[float, float] - modes: int - lambda_min: float - lambda_max: float - trange: float - n_save: int - expected_steps: int - - -class FamilyTable(TypedDict): - """Pulse and gas values of the M2 benchmark cases (``m2_cases.py``).""" - - gas: str - wavelength: float - tau_fwhm: float - energy: float - loss: bool - - -def _case_tables(case_name: str) -> tuple[CaseTable, FamilyTable]: - from m2_cases import CASES, FAMILIES - - return cast(CaseTable, CASES[case_name]), cast(FamilyTable, FAMILIES[case_name]) - - -def _metadata_int(metadata: Mapping[str, object], name: str, default: int) -> int: - """Read one integer-valued backend metadata item without a blind cast.""" - value = metadata.get(name, default) - if isinstance(value, bool) or not isinstance(value, int | str): - raise TypeError(f"metadata {name!r} is not integer-valued: {value!r}") - return int(value) - - -def _metadata_float(metadata: Mapping[str, object], name: str) -> float: - """Read one floating-point backend metadata item without a blind cast.""" - value = metadata.get(name, 0.0) - if isinstance(value, bool) or not isinstance(value, int | float): - raise TypeError(f"metadata {name!r} is not numeric: {value!r}") - return float(value) - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _march(session: Any) -> tuple[int, int, Any]: - """Advance a started session to its end, exporting every observation's field. - - Each observation's statistics arrive with it. Returns the attempts, the rejected - attempts among them and the last plane's field, in the internal convention. - """ - import numpy as np - - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - - identity = ExportRecipe(power=False, optics=False) - attempts = rejected = 0 - field = None - try: - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += history.n_steps - rejected += history.n_rejected - if isinstance(event, ObservationReady): - field = np.array(session.export(event.ticket, identity)) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - return attempts, rejected, field - - -def _worker( - backend_name: str, - case_name: str, - field_family: str, - save_field: Path | None, - mode: str, - trial_count: int, - planes: int | None, -) -> None: - if backend_name == "native": - # Kokkos loads first, the order documented for local applications. - __import__("solprop_native") - - import numpy as np - - from solprop.backends.node_chunk import plan_node_chunk - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.driver import Simulation - from solprop.exceptions import MaterialRangeWarning - - import warnings - - case, family = _case_tables(case_name) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", MaterialRangeWarning) - simulation = Simulation.from_capillary( - case["radius"], - case["length"], - family["gas"], - case["pressure"], - pressure_floor=0.0, - wavelength=family["wavelength"], - wavelength_limits=(case["lambda_min"], case["lambda_max"]), - trange=case["trange"], - fwhm=family["tau_fwhm"], - energy=family["energy"], - modes=case["modes"], - loss=family["loss"], - envelope=field_family == "envelope", - n_save=case["n_save"], - rtol=1e-6, - atol=1e-10, - backend=backend_name, - ) - problem = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - if planes is not None: - request = request.with_stepper( - dataclasses.replace( - request.stepper, save_planes=request.stepper.save_planes[:planes] - ) - ) - - def entrance() -> Start: - return Start.entrance(problem, noise=simulation.noise) - - construction_started = time.perf_counter() - backend: Any = select( - backend_name, - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - construction_seconds = time.perf_counter() - construction_started - # The chunk a native response pass holds; the reference's is the host plan's. - node_chunk = ( - int(backend.node_chunk) - if backend_name == "native" - else plan_node_chunk(problem).chunk - ) - dz = float(request.stepper.dz_init) - try: - memory_ledger = backend.memory_ledger() - _warm(start_session(backend, problem, request, entrance(), owns=False)) - if mode == "trial": - probe = backend.timing_probe(entrance()) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - probe.trial(0.0, dz) - field = probe.field() - attempts = 0 - rejected = 0 - trials = trial_count - else: - session = start_session(backend, problem, request, entrance(), owns=False) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - attempts, rejected, field = _march(session) - seconds = time.perf_counter() - started - trials = attempts - load_end = os.getloadavg()[0] - if not np.all(np.isfinite(field)): - raise RuntimeError("reduced M2 produced a non-finite field") - if save_field is not None: - np.save(save_field, field) - metadata = backend.metadata() - device_after = device_record(metadata) - finally: - backend.close() - - accepted = attempts - rejected - recorded_threads = _metadata_int(metadata, "thread_count", 0) - recorded_fftw_threads = _metadata_int(metadata, "fftw_threads", recorded_threads) - blas_backend = str(metadata.get("blas_backend", "not applicable")) - if blas_backend == "Accelerate": - blas_threads: int | str = "system-managed" - elif blas_backend == "OpenBLAS": - blas_threads = int(os.environ.get("OPENBLAS_NUM_THREADS", "1")) - elif blas_backend in {"cuBLAS", "rocBLAS"}: - blas_threads = "device-stream" - elif backend_name == "native": - blas_threads = "disabled" - else: - blas_threads = _metadata_int( - metadata, - "blas_threads", - int(os.environ.get("SOLPROP_BLAS_THREADS", "1")), - ) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "mode": mode, - "trials": trials, - "case": case_name, - "field_family": field_family, - "seconds": seconds, - "construction_seconds": construction_seconds, - "planning_seconds": _metadata_float(metadata, "planning_seconds"), - "memory_ledger": memory_ledger, - "memory_ledger_bytes": sum(memory_ledger.values()), - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "modes": problem.shape[0], - "nodes": problem.transform.n_nodes, - "node_chunk": node_chunk, - "components": problem.transform.n_pol * problem.shape[1], - "n_omega": problem.shape[-1], - "n_to": problem.grids.n_to, - "warmed": True, - "native_threads": recorded_threads, - "fftw_threads": recorded_fftw_threads, - "blas_threads": blas_threads, - "blas_thread_policy": str( - metadata.get("blas_thread_policy", "backend-managed") - ), - "gemm_provider": str(metadata.get("gemm_provider", "not applicable")), - "synthesis_provider": str(metadata.get("synthesis_provider", "not applicable")), - "projection_provider": str( - metadata.get("projection_provider", "not applicable") - ), - "blas_backend": blas_backend, - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _parse_csv(value: str) -> list[str]: - result = [part.strip() for part in value.split(",") if part.strip()] - if not result: - raise argparse.ArgumentTypeError("the list must not be empty") - return result - - -def _run( - backend: str, - threads: int, - blas_threads: int, - case_name: str, - field_family: str, - mode: str, - trial_count: int, - planes: int | None, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "OPENBLAS_NUM_THREADS": str(blas_threads), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - completed = subprocess.run( - [ - sys.executable, - __file__, - "--worker", - backend, - "--case", - case_name, - "--field-family", - field_family, - "--mode", - mode, - "--trial-count", - str(trial_count), - *(() if planes is None else ("--planes", str(planes))), - ], - check=True, - capture_output=True, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run isolated workers and emit median reduced-M2 JSON records.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument( - "--case", choices=("m2_reduced", "m2_full"), default="m2_reduced" - ) - parser.add_argument( - "--field-family", choices=("field_real", "envelope"), default="field_real" - ) - parser.add_argument( - "--save-field", - type=Path, - help="worker-only owning field output, written after the timed interval", - ) - parser.add_argument("--backends", type=_parse_csv, default=["numpy_ref", "native"]) - parser.add_argument("--threads", type=_parse_csv, default=["1", "8"]) - parser.add_argument("--blas-threads", type=int, default=1) - parser.add_argument("--repeats", type=int, default=1) - parser.add_argument( - "--mode", choices=("trial", "propagation"), default="propagation" - ) - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument( - "--planes", - type=int, - help="run the case's first planes only, to check the script", - ) - arguments = parser.parse_args() - if arguments.planes is not None and arguments.planes < 1: - parser.error("--planes must be positive") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.case, - arguments.field_family, - arguments.save_field, - arguments.mode, - arguments.trial_count, - arguments.planes, - ) - return - if arguments.save_field is not None: - parser.error("--save-field requires --worker") - if min(arguments.repeats, arguments.blas_threads, arguments.trial_count) < 1: - parser.error("--repeats, --blas-threads and --trial-count must be positive") - for backend in arguments.backends: - for thread_text in arguments.threads: - threads = int(thread_text) - if threads < 1: - parser.error("thread counts must be positive") - samples = [ - _run( - backend, - threads, - arguments.blas_threads, - arguments.case, - arguments.field_family, - arguments.mode, - arguments.trial_count, - arguments.planes, - ) - for _ in range(arguments.repeats) - ] - median = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - median)) - deviations = [abs(sample["seconds"] - median) for sample in samples] - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - "median_absolute_deviation": statistics.median(deviations), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m2_cases.py b/native/benchmarks/m2_cases.py deleted file mode 100644 index 7ccf4bde2..000000000 --- a/native/benchmarks/m2_cases.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""The M2 benchmark cases: vacuum-ultraviolet dispersive-wave emission in helium. - -The full case is the published RDW example (a 7.5 fs, 275 uJ pulse at 800 nm in a -1.5 m, 125 um-radius capillary with a pressure gradient from 0.8 bar to vacuum, four -modes); the reduced case is its short, two-mode companion. ``expected_steps`` records -the step count of each case's independent reference run. -""" - -from __future__ import annotations - -type Value = float | int | str | bool | tuple[float, float] - -RDW: dict[str, Value] = { - "gas": "He", - "wavelength": 800e-9, - "tau_fwhm": 7.5e-15, - "energy": 275e-6, - "loss": True, -} - -CASES: dict[str, dict[str, Value]] = { - "m2_full": { - "radius": 125e-6, - "length": 1.5, - "pressure": (0.8, 0.0), - "modes": 4, - "lambda_min": 90e-9, - "lambda_max": 4e-6, - "trange": 400e-15, - "n_save": 201, - "expected_steps": 4365, - }, - "m2_reduced": { - "radius": 125e-6, - "length": 1.0, - "pressure": (1.2, 0.0), - "modes": 2, - "lambda_min": 100e-9, - "lambda_max": 1.5e-6, - "trange": 200e-15, - "n_save": 51, - "expected_steps": 348, - }, -} - -FAMILIES: dict[str, dict[str, Value]] = dict.fromkeys(CASES, RDW) diff --git a/native/benchmarks/provider_sweep.py b/native/benchmarks/provider_sweep.py deleted file mode 100644 index f70cc13fa..000000000 --- a/native/benchmarks/provider_sweep.py +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Interleave isolated M2 or CtC runs across prebuilt backend variants.""" - -from __future__ import annotations - -import argparse -import json -import os -import statistics -import subprocess -import sys -from pathlib import Path -from typing import TypedDict, cast - - -class Variant(TypedDict): - """One backend label and its import-package root.""" - - label: str - package_root: str - - -class Result(TypedDict): - """One worker result augmented with sweep coordinates.""" - - variant: str - repetition: int - ordinal: int - seconds: float - load_start: float - load_end: float - - -def _variant(value: str) -> Variant: - label, separator, raw_path = value.partition("=") - if not separator or not label or not raw_path: - raise argparse.ArgumentTypeError( - "a Kokkos variant must be LABEL=/path/to/package-root" - ) - path = Path(raw_path).expanduser().resolve() - if not (path / "solprop_native").is_dir(): - raise argparse.ArgumentTypeError( - f"{path} does not contain a solprop_native package directory" - ) - return {"label": label, "package_root": str(path)} - - -def _run( - variant: Variant, - threads: int, - blas_threads: int, - workload: str, - case_name: str, - field_family: str, - mode: str, - trial_count: int, - node_chunk: int | None, -) -> dict[str, object]: - environment = os.environ.copy() - environment.update( - { - "KMP_BLOCKTIME": environment.get("KMP_BLOCKTIME", "50"), - "OPENBLAS_NUM_THREADS": str(blas_threads), - "OMP_DYNAMIC": environment.get("OMP_DYNAMIC", "FALSE"), - "OMP_PLACES": environment.get("OMP_PLACES", "threads"), - "OMP_PROC_BIND": environment.get("OMP_PROC_BIND", "close"), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - package_root = variant["package_root"] - previous = environment.get("PYTHONPATH") - environment["PYTHONPATH"] = ( - package_root if not previous else package_root + os.pathsep + previous - ) - command = [sys.executable, str(Path(__file__).with_name(f"{workload}.py"))] - command.extend(("--worker", "native", "--case", case_name)) - if workload == "m2": - command.extend(("--field-family", field_family)) - command.extend(("--mode", mode, "--trial-count", str(trial_count))) - if workload == "ctc": - if node_chunk is not None: - command.extend(("--node-chunk", str(node_chunk))) - completed = subprocess.run( - command, check=True, capture_output=True, text=True, env=environment - ) - return cast(dict[str, object], json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run a round-robin provider sweep and emit samples plus robust summaries.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--variant", - action="append", - type=_variant, - required=True, - help="LABEL=/path/to/package-root; repeat for each arm", - ) - parser.add_argument("--threads", type=int, default=8) - parser.add_argument("--workload", choices=("m2", "ctc"), default="m2") - parser.add_argument("--case", default="m2_reduced") - parser.add_argument( - "--field-family", choices=("field_real", "envelope"), default="field_real" - ) - parser.add_argument( - "--blas-threads", - type=int, - default=1, - help="process-start BLAS pool, independent of the Kokkos pool", - ) - parser.add_argument("--repeats", type=int, default=7) - parser.add_argument("--mode", choices=("trial", "propagation"), default="trial") - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument("--node-chunk", type=int) - parser.add_argument( - "--max-load", - type=float, - default=None, - help="fail if either one-minute load sample exceeds this value", - ) - arguments = parser.parse_args() - if ( - arguments.threads < 1 - or arguments.blas_threads < 1 - or arguments.repeats < 1 - or arguments.trial_count < 1 - ): - parser.error("thread counts and repeats must be positive") - if not arguments.case.startswith(f"{arguments.workload}_"): - parser.error("--case must belong to the selected workload") - if arguments.node_chunk is not None and arguments.node_chunk < 1: - parser.error("--node-chunk must be positive") - - samples: dict[str, list[Result]] = { - variant["label"]: [] for variant in arguments.variant - } - ordinal = 0 - for repetition in range(arguments.repeats): - offset = repetition % len(arguments.variant) - ordered = arguments.variant[offset:] + arguments.variant[:offset] - for variant in ordered: - measured = _run( - variant, - arguments.threads, - arguments.blas_threads, - arguments.workload, - arguments.case, - arguments.field_family, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - ) - sample: Result = { - **cast(Result, measured), - "variant": variant["label"], - "repetition": repetition, - "ordinal": ordinal, - } - ordinal += 1 - if arguments.max_load is not None and max( - sample["load_start"], sample["load_end"] - ) > cast(float, arguments.max_load): - raise RuntimeError( - f"load exceeded {arguments.max_load}: {json.dumps(sample)}" - ) - samples[variant["label"]].append(sample) - print(json.dumps({"kind": "sample", **sample}, sort_keys=True)) - - for label, values in samples.items(): - times = [value["seconds"] for value in values] - median = statistics.median(times) - print( - json.dumps( - { - "kind": "summary", - "variant": label, - "repeats": len(times), - "median_seconds": median, - "median_absolute_deviation": statistics.median( - abs(value - median) for value in times - ), - "minimum_seconds": min(times), - "maximum_seconds": max(times), - }, - sort_keys=True, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/native/ci/build-deps-windows.ps1 b/native/ci/build-deps-windows.ps1 index 1c226f60d..e9cd94cc3 100644 --- a/native/ci/build-deps-windows.ps1 +++ b/native/ci/build-deps-windows.ps1 @@ -88,7 +88,7 @@ $ompLibrary = ($omp -replace '\\', '/') + '/lib/libomp.lib' $openmp = @('-DOpenMP_C_FLAGS=-fopenmp', '-DOpenMP_C_LIB_NAMES=libomp', "-DOpenMP_libomp_LIBRARY=$ompLibrary") -# FFTW, double and single precision, static, with OpenMP. +# FFTW, double precision (the package computes in no other), static, with OpenMP. Get-Source "https://www.fftw.org/fftw-$fftwVersion.tar.gz" $fftwSha256 $fftwSimd = @('-DCMAKE_C_FLAGS=-march=x86-64-v2', '-DENABLE_SSE2=ON', '-DENABLE_AVX=ON') if ($arm) { @@ -105,19 +105,16 @@ if ($arm) { $text = $text.Replace('/* #undef HAVE_NEON */', "#define HAVE_NEON 1`n#define HAVE_ARMV8_CNTVCT_EL0 1") Set-Content $config $text -NoNewline } -foreach ($precision in @('double', 'float')) { - $build = Join-Path $work "fftw-build-$precision" - $flags = @('-S', "$work\fftw-$fftwVersion", '-B', $build, '-G', 'Ninja', - '-DCMAKE_C_COMPILER=clang', '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_POLICY_VERSION_MINIMUM=3.5', - '-DBUILD_SHARED_LIBS=OFF', '-DBUILD_TESTS=OFF', '-DENABLE_OPENMP=ON', - '-DENABLE_THREADS=OFF', - "-DCMAKE_INSTALL_PREFIX=$Prefix\fftw") + $fftwSimd + $openmp - if ($precision -eq 'float') { $flags += '-DENABLE_FLOAT=ON' } - cmake @flags; Invoke-Native "FFTW $precision configure" - cmake --build $build --parallel; Invoke-Native "FFTW $precision build" - cmake --install $build; Invoke-Native "FFTW $precision install" -} +$build = Join-Path $work 'fftw-build' +$flags = @('-S', "$work\fftw-$fftwVersion", '-B', $build, '-G', 'Ninja', + '-DCMAKE_C_COMPILER=clang', '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_POLICY_VERSION_MINIMUM=3.5', + '-DBUILD_SHARED_LIBS=OFF', '-DBUILD_TESTS=OFF', '-DENABLE_OPENMP=ON', + '-DENABLE_THREADS=OFF', + "-DCMAKE_INSTALL_PREFIX=$Prefix\fftw") + $fftwSimd + $openmp +cmake @flags; Invoke-Native 'FFTW configure' +cmake --build $build --parallel; Invoke-Native 'FFTW build' +cmake --install $build; Invoke-Native 'FFTW install' Copy-Item "$work\fftw-$fftwVersion\COPYING" "$licences\LICENSE-fftw.txt" # OpenBLAS, static, BLAS plus its C LAPACK translation (Kokkos Kernels needs crot_ and @@ -145,11 +142,9 @@ if ($arm) { Build-OpenBlas 'NEHALEM' "$Prefix\openblas-v2" } if ($arm) { - foreach ($library in @('fftw3.lib', 'fftw3f.lib')) { - # The version string names the codelets, as the smoke test reads it elsewhere. - if (-not (Select-String -Path "$Prefix\fftw\lib\$library" -Pattern '-neon' -SimpleMatch -Quiet)) { - throw "$library lacks the NEON codelets" - } + # The version string names the codelets, as the smoke test reads it elsewhere. + if (-not (Select-String -Path "$Prefix\fftw\lib\fftw3.lib" -Pattern '-neon' -SimpleMatch -Quiet)) { + throw 'fftw3.lib lacks the NEON codelets' } } Get-ChildItem "$Prefix\fftw\lib", "$Prefix\openblas\lib", "$omp\lib", "$omp\bin" | diff --git a/native/ci/build-fftw.sh b/native/ci/build-fftw.sh index 03d0a9221..22f5a242b 100755 --- a/native/ci/build-fftw.sh +++ b/native/ci/build-fftw.sh @@ -4,8 +4,8 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -# Build FFTW for a Linux or macOS wheel: double and single precision, shared, with -# its OpenMP and POSIX-threads libraries. +# Build FFTW for a Linux or macOS wheel: double precision (the package computes in no +# other), shared, with its OpenMP and POSIX-threads libraries. # # The wheel repair (auditwheel, delocate) copies these libraries into the package. # The common code is compiled for the wheel's CPU baseline, except on x86-64: there it @@ -14,8 +14,8 @@ # x86-64 FFTW's SSE2 and AVX codelets are chosen at run time (AVX2 and AVX-512 codelets were not a # uniform gain when measured, so they are left out). On Arm the cycle counter is # enabled, so that measured planning (FFTW_MEASURE) works; without one FFTW plans by -# estimate only. Both Arm wheels also enable FFTW's explicit NEON codelets in both -# precisions; compiler auto-vectorisation does not enable these implementations. +# estimate only. Both Arm wheels also enable FFTW's explicit NEON codelets; compiler +# auto-vectorisation does not enable these implementations. # # OpenMP: on Linux the compiler's own runtime (GCC's libgomp in the manylinux image); # on macOS the libomp of LIBOMP_PREFIX (native/ci/build-libomp.sh), the runtime the @@ -68,17 +68,13 @@ esac archive=$(fetch "https://www.fftw.org/fftw-$version.tar.gz" "$sha256" "$work") rm -rf "$work/fftw-$version" tar -xzf "$archive" -C "$work" -for precision in double single; do - build="$work/fftw-build-$precision" - rm -rf "$build" - mkdir -p "$build" - precision_args=(--disable-single) - [[ $precision == single ]] && precision_args=(--enable-single) - (cd "$build" && CFLAGS="$cflags" "$work/fftw-$version/configure" \ - "${configure_args[@]}" "${simd[@]}" "${precision_args[@]}" >configure.log) - make -C "$build" -j"$(jobs)" >"$build/make.log" - make -C "$build" install >"$build/install.log" -done +build="$work/fftw-build" +rm -rf "$build" +mkdir -p "$build" +(cd "$build" && CFLAGS="$cflags" "$work/fftw-$version/configure" \ + "${configure_args[@]}" "${simd[@]}" >configure.log) +make -C "$build" -j"$(jobs)" >"$build/make.log" +make -C "$build" install >"$build/install.log" # The wheel carries FFTW's licence (CMake's SOLPROP_NATIVE_BUNDLED_LICENCES). mkdir -p "$prefix/licences" cp "$work/fftw-$version/COPYING" "$prefix/licences/LICENSE-fftw.txt" diff --git a/native/ci/smoke.py b/native/ci/smoke.py index a9244dcb0..f141c3ab4 100644 --- a/native/ci/smoke.py +++ b/native/ci/smoke.py @@ -12,8 +12,9 @@ It checks that the package imports and passes ``check_build``, that it is the distribution build, that its version is the expected one, and that the libraries it loaded (FFT, BLAS, OpenMP) are the copies inside the installed wheel, not ones from -the build machine or the system. On Arm it checks that both precisions of the bundled -FFTW carry the NEON codelets. It then runs ``python -m solprop_native`` and checks +the build machine or the system, and that the wheel neither carries nor imports a +single-precision FFTW. On Arm it checks that the bundled FFTW carries the NEON codelets. +It then runs ``python -m solprop_native`` and checks that the build record it prints is JSON. """ @@ -42,30 +43,37 @@ def bundled(library: str, package: Path) -> bool: return any(path.is_relative_to(root) for root in roots) -def fftw_versions(fft_library: str) -> dict[str, str]: - """Return the version strings of the bundled FFTW's double and single libraries. +def fftw_version(fft_library: str) -> str: + """Return the bundled FFTW's version string, which names its codelet family. - FFTW names its codelet family in the version (``fftw-3.3.11-neon``). The single - precision library lies beside the double one under the repair's naming. + For example ``fftw-3.3.11-neon``. """ - single = sorted( - { - path.resolve() - for path in Path(fft_library).parent.glob("libfftw3f[-.]*") - if path.suffix not in {".a", ".la"} - } - ) - if len(single) != 1: - sys.exit(f"expected one single-precision FFTW beside {fft_library}: {single}") - versions = {} - for symbol, library in ( - ("fftw_version", fft_library), - ("fftwf_version", str(single[0])), - ): - handle = ctypes.CDLL(library) - address = ctypes.addressof(ctypes.c_char.in_dll(handle, symbol)) - versions[symbol] = ctypes.string_at(address).decode() - return versions + handle = ctypes.CDLL(fft_library) + address = ctypes.addressof(ctypes.c_char.in_dll(handle, "fftw_version")) + return ctypes.string_at(address).decode() + + +def single_precision_fftw(package: Path) -> list[str]: + """Return what in the installed wheel names a single-precision FFTW. + + The package computes in double precision only: the build points Kokkos-FFT's + single-precision library names at the double libraries + (``cmake/HostLibraries.cmake``), so no ``libfftw3f`` is linked or bundled, and the + extension imports no ``fftwf_`` symbol (an import's name is in the extension's + dynamic string table). + """ + found = [ + str(path) + for root in (package, package.parent / "solprop_native.libs") + if root.is_dir() + for path in root.rglob("*fftw3f*") + ] + found += [ + f"fftwf_ in {path.name}" + for path in package.glob("_core*") + if path.suffix in {".so", ".pyd"} and b"fftwf_" in path.read_bytes() + ] + return found def main() -> None: @@ -98,12 +106,14 @@ def main() -> None: and not bundled(library, package) ): sys.exit(f"{name} {library} is not the wheel's own copy (under {package})") + if single := single_precision_fftw(package): + sys.exit(f"the wheel carries a single-precision FFTW: {single}") # Compiling with -mcpu does not enable FFTW's NEON codelets; the recipe must. fftw = {} if platform.machine() in {"arm64", "aarch64"}: - fftw = fftw_versions(info.fft_library) - if scalar := [name for name, version in fftw.items() if "-neon" not in version]: - sys.exit(f"the bundled FFTW lacks NEON codelets: {fftw} ({scalar})") + fftw = {"fftw_version": fftw_version(info.fft_library)} + if "-neon" not in fftw["fftw_version"]: + sys.exit(f"the bundled FFTW lacks NEON codelets: {fftw}") if info.blas_backend == "OpenBLAS" and info.blas_threads is None: sys.exit("the bundled OpenBLAS reports no thread count") # Every library the wheel bundles travels with its licence. diff --git a/native/cmake/HostLibraries.cmake b/native/cmake/HostLibraries.cmake index 784120ac4..5d6d71bc8 100644 --- a/native/cmake/HostLibraries.cmake +++ b/native/cmake/HostLibraries.cmake @@ -219,14 +219,6 @@ if(SOLPROP_NATIVE_ENABLE_CUDA OR SOLPROP_NATIVE_ENABLE_HIP) endif() return() endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND - SOLPROP_NATIVE_ENABLE_OPENMP AND - SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "FFTW" AND - SOLPROP_NATIVE_FFTW_THREADING STREQUAL "OPENMP") - message(FATAL_ERROR - "IntelLLVM/FFTW CPU builds require FFTW_THREADING=PTHREADS. " - "Distribution OpenMP FFTW libraries link GNU OpenMP, not Intel's runtime.") -endif() if(_tpl_request STREQUAL "OFF" AND (NOT SOLPROP_NATIVE_BLAS_VENDOR STREQUAL "AUTO" OR SOLPROP_NATIVE_BLAS_LIBRARIES)) message(FATAL_ERROR @@ -237,11 +229,7 @@ if(NOT SOLPROP_NATIVE_BLAS_VENDOR STREQUAL "AUTO" AND message(FATAL_ERROR "Select BLAS_VENDOR or explicit BLAS_LIBRARIES, not both.") endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - set(_mkl_thread mkl_intel_thread) -else() - set(_mkl_thread mkl_gnu_thread) -endif() +set(_mkl_thread mkl_gnu_thread) if(NOT SOLPROP_NATIVE_ENABLE_OPENMP) set(_mkl_thread mkl_sequential) endif() @@ -476,8 +464,9 @@ if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "MKL_FFTW") HINTS "${SOLPROP_NATIVE_MKL_ROOT}" PATH_SUFFIXES include/fftw REQUIRED) set(FFTW_INCLUDE_DIRS "${SOLPROP_NATIVE_MKL_FFT_INCLUDE}" CACHE PATH "FFTW API headers" FORCE) # Kokkos FFT uses the same FFTW C API for both thread implementations. - # MKL exports both precisions and the planning-thread controls from its LP64 - # interface; its dependency libraries are linked after the FFT/BLAS targets. + # MKL exports the planning-thread controls from its LP64 interface; its + # dependency libraries are linked after the FFT/BLAS targets. The single-precision + # names are the same library, as with FFTW below. foreach(_component IN ITEMS DOUBLE_LIB FLOAT_LIB DOUBLE_OPENMP_LIB FLOAT_OPENMP_LIB) set(FFTW_${_component} "${SOLPROP_NATIVE_mkl_intel_lp64_LIBRARY}" CACHE FILEPATH "MKL FFTW compatibility library" FORCE) @@ -485,19 +474,12 @@ if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "MKL_FFTW") set(SOLPROP_NATIVE_MKL_FFT_LIBRARIES "${_mkl_libraries}") set(SOLPROP_NATIVE_HOST_FFT_BACKEND "MKL_FFTW") set(SOLPROP_NATIVE_HOST_FFT_PLANNER "MKL_ignores_FFTW_planning_flags") -elseif(SOLPROP_NATIVE_ENABLE_THREADS) - # Kokkos FFT links FFTW's threads library for the Threads host space. - set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_THREADS") elseif(SOLPROP_NATIVE_ENABLE_OPENMP) if(SOLPROP_NATIVE_FFTW_THREADING STREQUAL "PTHREADS") find_library(SOLPROP_NATIVE_FFTW_DOUBLE_PTHREADS fftw3_threads HINTS "${FFTW_ROOT}" PATH_SUFFIXES lib lib64 REQUIRED) - find_library(SOLPROP_NATIVE_FFTW_FLOAT_PTHREADS fftw3f_threads - HINTS "${FFTW_ROOT}" PATH_SUFFIXES lib lib64 REQUIRED) set(FFTW_DOUBLE_OPENMP_LIB "${SOLPROP_NATIVE_FFTW_DOUBLE_PTHREADS}" CACHE FILEPATH "FFTW pthread implementation of the thread API" FORCE) - set(FFTW_FLOAT_OPENMP_LIB "${SOLPROP_NATIVE_FFTW_FLOAT_PTHREADS}" - CACHE FILEPATH "FFTW pthread implementation of the thread API" FORCE) set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_PTHREADS") else() set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_OPENMP") @@ -505,3 +487,31 @@ elseif(SOLPROP_NATIVE_ENABLE_OPENMP) else() set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_SERIAL") endif() + +# solprop computes in double precision only. Kokkos-FFT's FindFFTW requires the +# single-precision libraries as well, though each of its fftwf_ calls sits behind an +# `if constexpr` on float that this package never instantiates. Name the double +# libraries for them, as the MKL branch above does, so that a build needs no +# single-precision FFTW and links none. FindFFTW keeps these cached results; a +# non-default FFTW is named with FFTW_ROOT (or FFTWDIR), as for FindFFTW. +if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "FFTW") + set(_suffixes "${CMAKE_FIND_LIBRARY_SUFFIXES}") + if(FFTW_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + set(_components DOUBLE_LIB:fftw3) + if(SOLPROP_NATIVE_ENABLE_OPENMP) + list(APPEND _components DOUBLE_OPENMP_LIB:fftw3_omp) + endif() + foreach(_component IN LISTS _components) + string(REPLACE ":" ";" _component "${_component}") + list(GET _component 0 _variable) + list(GET _component 1 _name) + find_library(FFTW_${_variable} NAMES ${_name} HINTS "${FFTW_ROOT}" "$ENV{FFTWDIR}" + PATH_SUFFIXES lib lib64 REQUIRED) + string(REPLACE "DOUBLE" "FLOAT" _float "${_variable}") + set(FFTW_${_float} "${FFTW_${_variable}}" CACHE FILEPATH + "The double-precision FFTW: the build makes no single-precision call" FORCE) + endforeach() + set(CMAKE_FIND_LIBRARY_SUFFIXES "${_suffixes}") +endif() diff --git a/native/cmake/SourceDigest.cmake b/native/cmake/SourceDigest.cmake index 9440f25cf..c94b8cbf4 100644 --- a/native/cmake/SourceDigest.cmake +++ b/native/cmake/SourceDigest.cmake @@ -34,6 +34,7 @@ string(SHA256 digest "${record}") file(CONFIGURE OUTPUT "${OUTPUT}" @ONLY CONTENT "// Generated by cmake/SourceDigest.cmake: the digest of what the extension is compiled from. namespace solprop_native { +// NOLINTNEXTLINE(misc-use-internal-linkage): bindings.cpp declares and reads it const char* source_digest() noexcept { return \"@digest@\"; } } // namespace solprop_native ") diff --git a/native/cmake/VectorMath.cmake b/native/cmake/VectorMath.cmake index 6ef397cb3..b1d83ef37 100644 --- a/native/cmake/VectorMath.cmake +++ b/native/cmake/VectorMath.cmake @@ -85,13 +85,12 @@ function(_solprop_native_add_sleef) FetchContent_MakeAvailable(sleef) include(ExternalProject) # SLEEF requires a C and a C++ compiler of one version: the C driver beside this - # build's C++ compiler (clang beside clang++, gcc beside g++, icx beside icpx), else + # build's C++ compiler (clang beside clang++, gcc beside g++), else # the C compiler this project found. get_filename_component(_cxx_directory "${CMAKE_CXX_COMPILER}" DIRECTORY) get_filename_component(_cxx_name "${CMAKE_CXX_COMPILER}" NAME) string(REGEX REPLACE "clang\\+\\+" "clang" _c_name "${_cxx_name}") string(REGEX REPLACE "g\\+\\+" "gcc" _c_name "${_c_name}") - string(REGEX REPLACE "^icpx" "icx" _c_name "${_c_name}") set(_c_compiler "${_cxx_directory}/${_c_name}") if(_c_name STREQUAL _cxx_name OR NOT EXISTS "${_c_compiler}") set(_c_compiler "${CMAKE_C_COMPILER}") diff --git a/native/include/solprop_native/adams_march.hpp b/native/include/solprop_native/adams_march.hpp index b0f7904dc..f7c586de5 100644 --- a/native/include/solprop_native/adams_march.hpp +++ b/native/include/solprop_native/adams_march.hpp @@ -4,6 +4,8 @@ #pragma once +#include "solprop_native/watch.hpp" + #include #include #include @@ -13,8 +15,6 @@ #include #include -#include "solprop_native/watch.hpp" - /* * The exponential Adams march executed natively (solprop SEP 39 phase 2). * diff --git a/native/include/solprop_native/adams_phi.hpp b/native/include/solprop_native/adams_phi.hpp index 9e9b19e22..d00f7ce07 100644 --- a/native/include/solprop_native/adams_phi.hpp +++ b/native/include/solprop_native/adams_phi.hpp @@ -81,7 +81,7 @@ adams_phi_fixed(const Complex x, const double magnitude, const Complex exp_x, // 1/n for the series' divisors m + order, the doubles the division rounds to. static_assert(adams_series_terms + 2 * adams_max_samples <= 40); constexpr double inverse_divisor[40] = { - 0.0, 1.0 / 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, + 0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, 1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0, 1.0 / 13.0, 1.0 / 14.0, 1.0 / 15.0, 1.0 / 16.0, 1.0 / 17.0, 1.0 / 18.0, 1.0 / 19.0, 1.0 / 20.0, 1.0 / 21.0, 1.0 / 22.0, 1.0 / 23.0, @@ -168,7 +168,7 @@ inline constexpr double adams_lane_reciprocal[adams_max_samples + 2] = { 1.0 / 120.0, 1.0 / 720.0, 1.0 / 5040.0, 1.0 / 40320.0, 1.0 / 362880.0}; inline constexpr double adams_lane_inverse_divisor[adams_series_terms + adams_max_samples + 1] = { - 0.0, 1.0 / 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, + 0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, 1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0, 1.0 / 13.0, 1.0 / 14.0, 1.0 / 15.0, 1.0 / 16.0, 1.0 / 17.0, 1.0 / 18.0, 1.0 / 19.0, 1.0 / 20.0, 1.0 / 21.0, 1.0 / 22.0, 1.0 / 23.0, 1.0 / 24.0, 1.0 / 25.0, 1.0 / 26.0, 1.0 / 27.0, @@ -223,6 +223,9 @@ struct AdamsLanes { double dr[B], di[B]; double defect[3][B]; }; +// The lane passes spell their branches as signed 64-bit masks and bitwise selections, +// the form every x86-64 target vectorises. +// NOLINTBEGIN(bugprone-signed-bitwise, readability-implicit-bool-conversion): lane masks // The moments' argument and its branch, halvings and series argument, as // `adams_moment_argument`, `adams_phi` and `adams_phi_argument` form them, with the @@ -257,13 +260,10 @@ inline void adams_lanes_classify(AdamsLanes& lanes, const ExponentialWeights& we const double square = rr + ii; const std::int64_t zero = static_cast(lanes.ar[b] == 0.0) & static_cast(lanes.ai[b] == 0.0); - const std::int64_t large = static_cast(std::isgreater(square, 9.0)); - const std::int64_t above_quarter = - static_cast(std::isgreater(square, 0.25)); - const std::int64_t above_one = - static_cast(std::isgreater(square, 1.0)); - const std::int64_t above_four = - static_cast(std::isgreater(square, 4.0)); + const auto large = static_cast(std::isgreater(square, 9.0)); + const auto above_quarter = static_cast(std::isgreater(square, 0.25)); + const auto above_one = static_cast(std::isgreater(square, 1.0)); + const auto above_four = static_cast(std::isgreater(square, 4.0)); const std::int64_t uncertain = static_cast(!std::islessequal(square, finite)) | static_cast( @@ -279,7 +279,7 @@ inline void adams_lanes_classify(AdamsLanes& lanes, const ExponentialWeights& we // zero 0, large 1, series 2, and 2^-squarings, each factor exact; spelt as // arithmetic and selections of bits, which every x86-64 target vectorises. const std::int64_t branch = (1 - zero) * (2 - large); - const std::int64_t series = static_cast(branch == adams_lane_series); + const auto series = static_cast(branch == adams_lane_series); lanes.branch[b] = branch; lanes.squarings[b] = series * (above_quarter + above_one + above_four); scale[b] = adams_lane_select(above_quarter != 0, 0.5, 1.0) * @@ -329,7 +329,10 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { constexpr double stop = quarter_ulp * quarter_ulp; // The series: every lane takes its terms until its own stopping test holds; a lane // that has stopped keeps its sum and carries a zero term. - double term_r[B], term_i[B], sum_r[B], sum_i[B]; + double term_r[B]; + double term_i[B]; + double sum_r[B]; + double sum_i[B]; std::int64_t live[B]; const double first = adams_lane_reciprocal[order]; for (std::size_t b = 0; b < count; ++b) { @@ -412,7 +415,8 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { double half = 1.0; // 2^-k, exact for (std::size_t k = 1; k <= order; ++k) { half *= 0.5; - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < count; ++b) { const double rr = lanes.qr[0][b] * lanes.qr[k][b]; const double ii = lanes.qi[0][b] * lanes.qi[k][b]; @@ -440,7 +444,10 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { // Large arguments: phi_k = (phi_{k-1} - 1/(k-1)!) / x, the inverse by // `strict_divide`'s scaled algorithm. if (lanes.any_large) { - double inverse_r[B], inverse_i[B], last_r[B], last_i[B]; + double inverse_r[B]; + double inverse_i[B]; + double last_r[B]; + double last_i[B]; for (std::size_t b = 0; b < count; ++b) { const double s = Kokkos::fabs(lanes.ar[b]) + Kokkos::fabs(lanes.ai[b]); const double numerator_r = 1.0 / s; @@ -488,6 +495,8 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { } } } +// NOLINTEND(bugprone-signed-bitwise, readability-implicit-bool-conversion) + // e^argument for the large branch into `gr`/`gi`: the flow `fr`/`fi`, or under a // carrier frame the flow times exp(-sigma), as `adams_moment_phi` forms it. Strict. SOLPROP_NATIVE_STRICT_FUNCTION diff --git a/native/include/solprop_native/build.hpp b/native/include/solprop_native/build.hpp new file mode 100644 index 000000000..b9cd978c9 --- /dev/null +++ b/native/include/solprop_native/build.hpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +/** The build's facts as constants, for `if constexpr` where both branches compile in + every build (CPU, CUDA and HIP); code that names a device or host-only facility + keeps its `#if`. */ +namespace solprop_native::build { + +#ifdef KOKKOS_ENABLE_CUDA +inline constexpr bool cuda = true; +#else +inline constexpr bool cuda = false; +#endif + +#ifdef KOKKOS_ENABLE_HIP +inline constexpr bool hip = true; +#else +inline constexpr bool hip = false; +#endif + +/** A CUDA or HIP build: the kernels' execution space is a device. */ +inline constexpr bool device = cuda || hip; + +/** A host build: the kernels run on the CPU. */ +inline constexpr bool host = !device; + +#ifdef KOKKOS_ENABLE_OPENMP +inline constexpr bool openmp = true; +#else +inline constexpr bool openmp = false; +#endif + +} // namespace solprop_native::build diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index 00f693695..6208a5011 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -4,11 +4,11 @@ #pragma once +#include "solprop_native/launch.hpp" + #include #include -#include "solprop_native/launch.hpp" - #include #include #include @@ -138,7 +138,7 @@ inline constexpr std::size_t occupancy_bucket_limit = 128; // checks only the first: a value that fills the block passes its check and the dispatch // then faults on an AMD Instinct MI325X ("Illegal opcode in command stream"). So a // reading there takes two passes, leaving half the block free. -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP inline constexpr std::size_t occupancy_reduction_slots = 64; #else inline constexpr std::size_t occupancy_reduction_slots = occupancy_bucket_limit; @@ -151,7 +151,7 @@ struct OccupancySums { return *this; } }; -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP static_assert((64 + 2) * sizeof(OccupancySums) <= 48 * 1024); #endif @@ -471,238 +471,303 @@ inline constexpr std::size_t plasma_max_taps = 9; /** The six parts of SEP 18's run diagnostics, in solprop's order. */ inline constexpr std::size_t plasma_diagnostic_parts = 6; -/** Fixed, Python-free problem data uploaded once by a native state. */ +/** Fixed, Python-free problem data uploaded once by a native state, grouped as + solprop's kernel contract groups it. The adapter fills it field by field (`lower` in + solprop_native's backend); `validate` checks presence and extents before a state + allocates anything. */ struct Problem { - std::size_t n_x{}, n_y{}; - std::vector spatial_k_squared, spatial_k_squared_low, spatial_numerator, - spatial_clamped; - std::vector spatial_kperp_sq, spatial_kperp_sq_low, spatial_frame, - spatial_spectral_rate; - // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only - // by `absorbed_power`; the operator takes the total. - std::vector spatial_user_rate; - std::vector spatial_absorber_rate; - // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per - // column, added to the field attenuation on active bins; zeros leave the operator. - std::vector spatial_kperp_rate; - double spatial_evanescent_rate{}; - // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y - // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics read; - // and the slots of the problem's own far-field request, which size the band partials - // at construction (zero for none). Empty and zero elsewhere. - std::vector spatial_k_x, spatial_k_y; - std::size_t spatial_far_field_slots{}; - // solprop's hybrid planar kind (kernel schema 15): n_p slab - // channels in x on the real (n_x, n_p) synthesis and (n_p, n_x) projection, times a - // raw DFT in y. Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, - // and nodes (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y - // rows. Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, - // and spatial_absorber_rate is then per column. Zero n_p and empty kappa elsewhere. - std::size_t n_p{}; - std::vector spatial_kappa_sq; - // solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, - // the entrance's material squares (n_omega) and transverse squares (n_modes) whose - // root is solprop's `longitudinal_root`; `flux_table` when the material square - // varies along z and is uploaded as a `TableKind::flux` table of n_omega values. A - // separable run's measure is its own operator's (these stay empty). `flux_closure` - // allocates the magnetic closure's buffers at construction, for a request with a - // local flux statistic. - std::vector flux_k_squared, flux_k_squared_low; - std::vector flux_kperp_sq, flux_kperp_sq_low; - bool flux_table{}; - bool flux_closure{}; - /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ - [[nodiscard]] bool separable_geometry() const noexcept { return n_x != 0; } - [[nodiscard]] bool slab_geometry() const noexcept { return n_p != 0; } - [[nodiscard]] bool free_geometry() const noexcept { return n_x != 0 && n_p == 0; } - std::size_t n_omega{}; - std::size_t n_t{}; - std::size_t n_to{}; - std::size_t n_modes{1}; - std::size_t n_pol_state{1}; - std::size_t n_pol_transform{1}; - std::size_t n_nodes{1}; - std::size_t node_chunk{1}; + /** The state's extents and field family. */ + struct Dims { + std::size_t n_omega{}; + std::size_t n_t{}; + std::size_t n_to{}; + std::size_t n_modes{1}; + std::size_t n_pol_state{1}; + std::size_t n_pol_transform{1}; + std::size_t n_nodes{1}; + std::size_t node_chunk{1}; + FieldFamily family{FieldFamily::field_real}; + }; + /** The frequency and time grids, their windows and masks. */ + struct Grid { + std::vector crop_index; + std::vector active_mask; + std::vector norm_mask; + std::vector omega_window; + std::vector omega; + std::vector power_weight; + std::vector time_window; + std::vector time_window_derivative; + std::vector carrier; + double dt_fine{}; + double dt{}; + }; + /** The separable geometries' operator (free space and the hybrid planar kind) and a + radial run's absorber and on-axis rows. */ + struct Spatial { + std::size_t n_x{}, n_y{}; + std::vector k_squared, k_squared_low, numerator, clamped; + std::vector kperp_sq, kperp_sq_low, frame, spectral_rate; + // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only + // by `absorbed_power`; the operator takes the total. + std::vector user_rate; + std::vector absorber_rate; + // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per + // column, added to the field attenuation on active bins; zeros leave the operator. + std::vector kperp_rate; + double evanescent_rate{}; + // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y + // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics + // read; and the slots of the problem's own far-field request, which size the band + // partials at construction (zero for none). Empty and zero elsewhere. + std::vector k_x, k_y; + std::size_t far_field_slots{}; + // solprop's hybrid planar kind (kernel schema 15): n_p slab channels in x on the + // real (n_x, n_p) synthesis and (n_p, n_x) projection, times a raw DFT in y. + // Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, and nodes + // (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y rows. + // Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, and + // absorber_rate is then per column. Zero n_p and empty kappa elsewhere. + std::size_t n_p{}; + std::vector kappa_sq; + // A radial run's spatial absorber and its on-axis statistics' rows. + std::vector absorber_matrix; + std::vector spectral_weights; + std::vector on_axis; + }; + /** solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, + the entrance's material squares (n_omega) and transverse squares (n_modes) whose + root is solprop's `longitudinal_root`; `table` when the material square varies + along z and is uploaded as a `TableKind::flux` table of n_omega values. A + separable run's measure is its own operator's (these stay empty). `closure` + allocates the magnetic closure's buffers at construction, for a request with a + local flux statistic. */ + struct Flux { + std::vector k_squared, k_squared_low; + std::vector kperp_sq, kperp_sq_low; + bool table{}; + bool closure{}; + }; + /** The Runge-Kutta tableau and the march's own choices. */ + struct Stepper { + std::size_t n_stages{}; + std::vector tableau_matrix; + std::vector tableau_nodes; + std::vector solution_weights; + std::vector error_weights; + bool first_same_as_last{}; + bool weighted_controller{}; + bool modified_noise{}; + // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it + // no Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, + // recycled stage, whole-step propagator, stage inverse); the accumulator register + // is, the stage register unless it is elided (solprop's + // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 + // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). + bool low_storage{}; + bool low_storage_stage_register{true}; + std::size_t low_storage_flows{}; + // Whether a separable route's first input crosses to the transverse scratch + // through tiles of team scratch, as a device's does. + bool low_storage_tiled_input{}; + }; + /** The linear operator's fixed terms, row-major (terms, columns, n_omega). solprop + SEP 29 G1: Phi or the prefactor is the general z table when `phi_table` + (`prefactor_table`); its terms are then `n_phi_terms` (`n_prefactor_terms`) slots + that `table_evaluate` fills with the table's values at a trial's positions, and + `phi_terms` (`prefactor_terms`) is empty. */ + struct Linear { + std::size_t n_phi_terms{}; + std::size_t n_prefactor_terms{}; + bool phi_table{}; + bool prefactor_table{}; + std::vector phi_terms; + std::vector prefactor_terms; + }; + /** The temporal absorber, applied in each stage or split around the step. */ + struct Absorber { + bool in_stage{}; + bool split_step{}; + std::vector fine; + std::vector coarse; + }; + /** The modal synthesis and projection onto the nodes. */ + struct Transform { + std::vector synthesis_matrix; + std::vector projection_matrix; + std::vector projection_coarse_matrix; + double synthesis{}; + double projection{}; + }; + /** The instantaneous Kerr response. */ + struct Kerr { + bool nonlinear{}; + double coefficient{}; + double self_factor{}; + double conjugate_factor{}; + double harmonic_factor{}; + bool take_real{}; + }; + /** The field and envelope plasma columns and their rate tables. */ + struct Plasma { + bool enabled{}; + bool envelope{}; + // solprop's private, test-only gated plasma benchmark: the field plasma's current + // enters as w(t) times its running integral, added to the local polarisation, + // instead of the held current's two collared spectra. Never set by a user run. + bool gated{}; + bool stash{}; + // Channel-major 4 x 512 split angular rules and coarse-axis current factors. + std::vector envelope_rule_nodes, envelope_rule_weights, + envelope_drude_factor; + std::vector envelope_loss_factor; + double charge_mass_ratio{}; + double ionisation_potential{}; + double log_field_min{}; + double d_log_field{}; + double field_min{}; + double field_max{}; + double rate_max{}; + std::size_t rate_intervals{}; + // Row-major (4, rate_intervals). + std::vector rate_coefficients; + std::size_t n_channels{1}; + // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral + // rule, one per problem and shared by every channel. Each running integral is the + // cumulative trapezoid of the integrand filtered by the symmetric taps + // (c_0, ..., c_K), centre first, with the window's ends closed by point + // reflection, and the exposure floored at zero where the fraction is formed. The + // trapezoid rule is `band == false` with the one identity tap, and runs the + // version-24 column unchanged. + bool band{}; + std::vector taps{1.0}; + [[nodiscard]] std::size_t half_width() const noexcept { + return taps.empty() ? 0 : taps.size() - 1; + } + // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, + // interval count, coefficient offset, charge/mass ratio, ionisation potential. + std::vector channel_parameters, channel_coefficients; + }; + /** The delayed Raman response. */ + struct Raman { + bool enabled{}; + bool thg{}; + bool tensor_capable{}; + std::size_t n_terms{}; + double fraction{}; + double epsilon0{}; + // Row-major (n_terms, 2 n_to) and (n_terms, 2). + std::vector undamped; + std::vector tensor_weights; + std::size_t n_components{1}; + std::vector row_component, component_fraction; + }; + /** solprop's general modal route. Row-major: synthesis (map terms, nodes, + components, modes, n_omega); projection (map terms, modes, nodes, components, + n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, terms, C, C, C, + C); delayed spectrum (responses, terms, n_to + 1); node response (nodes), the + owning response plus one and zero for none. Empty otherwise. The map terms are + map_intervals x map_n_deg Chebyshev coefficients along z, one for a constant + guide; a moving guide's are combined per stage over + TrialScalars::transform_weights. */ + struct General { + bool enabled{}; + bool real_form{}; + std::size_t map_intervals{1}, map_n_deg{1}; + std::size_t components{}, nodes{}, node_chunk{}; + std::size_t responses{}, terms{}; + double pad_scale{}, epsilon0{}; + std::vector synthesis, projection, frequency_factor, delayed_spectrum; + std::vector tensor, delayed_tensor; + std::vector node_response; + }; + /** solprop's linear.coupled_flow. `slot` (n_omega) numbers the bins a section's + generator is non-zero at 1, 2, ... in increasing frequency, zero elsewhere. With + an explicit residual (terms > 0), row-major frozen blocks (sections, active, + modes, modes) and residual terms (terms, active, modes, modes); both empty + otherwise. The actions are loaded per step. + + The actions' native formation, all present or all empty: each section's + eigenbasis of its Schur factor, row-major left `W` and right `Y` (sections, + active, modes, modes) and eigenvalues (sections, active, modes); the route + (sections, active), zero where a bin takes its eigenbasis and 1, 2, ... + numbering in row-major order the bins formed by scaling and squaring, whose Schur + factors `Z` and `T` are the rows of the fallback arrays (fallback, modes, + modes). */ + struct Flow { + bool enabled{}; + std::size_t active{}, sections{}, terms{}; + std::vector slot; + std::vector frozen, residual_terms; + std::vector left, right, eigenvalues; + std::vector route; + std::vector fallback_vectors, fallback_triangular; + }; + /** solprop SEP 6's frequency-dependent electronic response + (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out + by the adapter as solprop's `dispersive_terms_layout` lays it out + (`dispersive.hpp`). The form is 0 (a field grid's full cubic), 1 (the + number-conserving sector) or 2 (an envelope's full cubic). Row-major on the + leaf's spectral layout of `bins` (n_to / 2 + 1 on a field grid, n_to on an + envelope): the input multipliers (fields, bins), each output term's multiplier + (terms, bins) with its product row and factor, the static coefficients and the + polynomial's pair constants (channels, 2) per channel, the groups as offsets + (groups + 1) into members (channels), each symmetric mixture's channels' `g^4` + (channels, bins) and the input and output scales (bins). On the general modal + route (SEP 6 Phase 3b) each group's key is its node region (groups), the + polynomial forms one cube per distinct key, and every general node's region + (general nodes) says where a channel acts; both are empty elsewhere, where every + channel acts at every node. Empty otherwise. */ + struct Dispersive { + bool enabled{}; + bool symmetric{}; + int form{}; + std::vector inputs, term_multipliers, coefficients, pair_constants, + quartics, input_scale, output_scale; + std::vector term_rows, term_channels, group_offsets, group_members, + group_keys, node_region; + }; + + Dims dims; + Grid grid; + Spatial spatial; + Flux flux; + Stepper stepper; + Linear linear; + Absorber absorber; + Transform transform; + Kerr kerr; + Plasma plasma; + Raman raman; + General general; + Flow flow; + Dispersive dispersive; // How the workspace's CPU operations thread (solprop SEP 46): set by the caller, // refused when unset (`require_cpu_execution`). CpuExecution cpu_execution{}; // How the workspace plans its host transforms (solprop SEP 46 phase 3). FftPlanning fft_planning{}; - std::size_t n_stages{}; - // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it no - // Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, - // recycled stage, whole-step propagator, stage inverse); the accumulator register - // is, the stage register unless it is elided (solprop's - // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 - // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). - bool low_storage{}; - bool low_storage_stage_register{true}; - std::size_t low_storage_flows{}; - // Whether a separable route's first input crosses to the transverse scratch through - // tiles of team scratch, as a device's does. - bool low_storage_tiled_input{}; - std::size_t n_phi_terms{}; - std::size_t n_prefactor_terms{}; - // solprop SEP 29 G1: Phi or the prefactor is the general z table. Its terms are then - // `n_phi_terms` (or `n_prefactor_terms`) slots that `table_evaluate` fills with the - // table's values at a trial's positions, and `phi_terms` (`prefactor_terms`) is - // empty. - bool phi_table{}; - bool prefactor_table{}; - FieldFamily family{FieldFamily::field_real}; - bool nonlinear{}; - bool modified_noise{}; - bool in_stage_absorber{}; - bool split_step_absorber{}; - bool first_same_as_last{}; - std::vector crop_index; - std::vector active_mask; - std::vector norm_mask; - std::vector omega_window; - std::vector omega; - std::vector power_weight; - std::vector time_window; - std::vector time_window_derivative; - std::vector absorber_fine; - std::vector absorber_coarse; - std::vector spatial_absorber_matrix; - std::vector spatial_spectral_weights; - std::vector spatial_on_axis; - std::vector carrier; - std::vector phi_terms; - std::vector prefactor_terms; - std::vector synthesis_matrix; - std::vector projection_matrix; - std::vector projection_coarse_matrix; - std::vector tableau_matrix; - std::vector tableau_nodes; - std::vector solution_weights; - std::vector error_weights; - double synthesis{}; - double projection{}; - double kerr_coefficient{}; - double self_factor{}; - double conjugate_factor{}; - double harmonic_factor{}; - bool take_real{}; - bool plasma{}; - bool envelope_plasma{}; - // solprop's private, test-only gated plasma benchmark: the field plasma's current - // enters as w(t) times its running integral, added to the local polarisation, - // instead of the held current's two collared spectra. Never set by a user run. - bool plasma_gated{}; - // Channel-major 4 x 512 split angular rules and coarse-axis current factors. - std::vector envelope_rule_nodes, envelope_rule_weights, envelope_drude_factor; - std::vector envelope_loss_factor; - bool raman{}; - bool raman_thg{}; - bool raman_tensor_capable{}; - bool stash_plasma{}; - bool weighted_controller{}; - double dt_fine{}; - double dt{}; - std::size_t n_raman_terms{}; - double raman_fraction{}; - double raman_epsilon0{}; - std::vector raman_undamped; - std::vector raman_tensor_weights; - double charge_mass_ratio{}; - double ionisation_potential{}; - double log_field_min{}; - double d_log_field{}; - double field_min{}; - double field_max{}; - double rate_max{}; - std::size_t rate_intervals{}; - std::vector rate_coefficients; - std::size_t n_channels{1}; - // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral - // rule, one per problem and shared by every channel. Each running integral is the - // cumulative trapezoid of the integrand filtered by the symmetric taps - // (c_0, ..., c_K), centre first, with the window's ends closed by point reflection, - // and the exposure floored at zero where the fraction is formed. The trapezoid rule - // is `plasma_band == false` with the one identity tap, and runs the version-24 - // column unchanged. - bool plasma_band{}; - std::vector plasma_taps{1.0}; - [[nodiscard]] std::size_t plasma_half_width() const noexcept { - return plasma_taps.empty() ? 0 : plasma_taps.size() - 1; + + /** Refuse, naming the field, data whose presence or extents disagree with the + problem's own flags and extents: the checks that need no execution space. */ + void validate() const; + + /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ + [[nodiscard]] bool separable_geometry() const noexcept { return spatial.n_x != 0; } + [[nodiscard]] bool slab_geometry() const noexcept { return spatial.n_p != 0; } + [[nodiscard]] bool free_geometry() const noexcept { + return spatial.n_x != 0 && spatial.n_p == 0; + } + [[nodiscard]] std::size_t n_columns() const noexcept { + return dims.n_modes * dims.n_pol_state; } - // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, - // interval count, coefficient offset, charge/mass ratio, ionisation potential. - std::vector channel_parameters, channel_coefficients; - std::size_t n_raman_components{1}; - std::vector raman_row_component, raman_component_fraction; - // solprop's general modal route. Row-major: synthesis (map terms, - // nodes, components, modes, n_omega); projection (map terms, modes, nodes, - // components, n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, - // terms, C, C, C, C); delayed spectrum (responses, terms, n_to + 1); node response - // (nodes), the owning response plus one and zero for none. Empty otherwise. The - // map terms are general_map_intervals x general_map_n_deg Chebyshev coefficients - // along z, one for a constant guide; a moving guide's are combined per stage over - // TrialScalars::transform_weights. - bool general_modal{}; - bool general_real_form{}; - std::size_t general_map_intervals{1}, general_map_n_deg{1}; - std::size_t general_components{}, general_nodes{}, general_node_chunk{}; - std::size_t general_responses{}, general_terms{}; - double general_pad_scale{}, general_epsilon0{}; - std::vector general_synthesis, general_projection, general_frequency_factor, - general_delayed_spectrum; - std::vector general_tensor, general_delayed_tensor; - std::vector general_node_response; - // solprop's linear.coupled_flow. flow_slot (n_omega) - // numbers the bins a section's generator is non-zero at 1, 2, ... in increasing - // frequency, zero elsewhere. With an explicit residual (flow_terms > 0), row-major - // frozen blocks (sections, active, modes, modes) and residual terms (terms, active, - // modes, modes); both empty otherwise. The actions are loaded per step. - bool coupled_flow{}; - std::size_t flow_active{}, flow_sections{}, flow_terms{}; - std::vector flow_slot; - std::vector flow_frozen, flow_residual_terms; - // The actions' native formation, all present or all - // empty: each section's eigenbasis of its Schur factor, row-major left `W` and - // right `Y` (sections, active, modes, modes) and eigenvalues (sections, active, - // modes); the route (sections, active), zero where a bin takes its eigenbasis and - // 1, 2, ... numbering in row-major order the bins formed by scaling and squaring, - // whose Schur factors `Z` and `T` are the rows of the fallback arrays (fallback, - // modes, modes). - std::vector flow_left, flow_right, flow_eigenvalues; - std::vector flow_route; - std::vector flow_fallback_vectors, flow_fallback_triangular; - - // solprop SEP 6's frequency-dependent electronic response - // (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out by - // the adapter as solprop's `dispersive_terms_layout` lays it out (`dispersive.hpp`). - // The form is 0 (a field grid's full cubic), 1 (the number-conserving sector) or 2 - // (an envelope's full cubic). Row-major on the leaf's spectral layout of `bins` (n_to - // / 2 + 1 on a field grid, n_to on an envelope): the input multipliers (fields, - // bins), each output term's multiplier (terms, bins) with its product row and factor, - // the static coefficients and the polynomial's pair constants (channels, 2) per - // channel, the groups as offsets (groups + 1) into members (channels), each symmetric - // mixture's channels' `g^4` (channels, bins) and the input and output scales (bins). - // On the general modal route (SEP 6 Phase 3b) each group's key is its node region - // (groups), the polynomial forms one cube per distinct key, and every general node's - // region (general_nodes) says where a channel acts; both are empty elsewhere, where - // every channel acts at every node. Empty otherwise. - bool dispersive{}; - bool dispersive_symmetric{}; - int dispersive_form{}; - std::vector dispersive_inputs, dispersive_term_multipliers, - dispersive_coefficients, dispersive_pair_constants, dispersive_quartics, - dispersive_input_scale, dispersive_output_scale; - std::vector dispersive_term_rows, dispersive_term_channels, - dispersive_group_offsets, dispersive_group_members, dispersive_group_keys, - dispersive_node_region; - - [[nodiscard]] std::size_t n_columns() const noexcept { return n_modes * n_pol_state; } [[nodiscard]] std::size_t effective_node_pol() const noexcept { - return n_pol_transform * n_pol_state; + return dims.n_pol_transform * dims.n_pol_state; } [[nodiscard]] std::size_t raman_drives() const noexcept { if (effective_node_pol() == 2) - for (std::size_t index = 1; index < raman_tensor_weights.size(); index += 2) - if (raman_tensor_weights[index] != 0.0) + for (std::size_t index = 1; index < raman.tensor_weights.size(); index += 2) + if (raman.tensor_weights[index] != 0.0) return 3; return 1; } @@ -1090,6 +1155,12 @@ class GeneralState { void close() noexcept; private: + /** `function(workspace, arguments...)` inside the workspace's CPU team scope + (`CpuTeamScope`); defined and used in general.cpp alone, so that no other unit + sees the workspace class. */ + template + decltype(auto) with_team(Function function, Arguments&&... arguments) const; + std::unique_ptr workspace_; }; diff --git a/native/include/solprop_native/coupled_flow.hpp b/native/include/solprop_native/coupled_flow.hpp index 5aeca605b..8faf107b5 100644 --- a/native/include/solprop_native/coupled_flow.hpp +++ b/native/include/solprop_native/coupled_flow.hpp @@ -46,7 +46,7 @@ class CoupledFlowAction { using IndexView = Kokkos::View; using IndexView2 = Kokkos::View; - CoupledFlowAction(const Problem& problem, const ExecutionSpace& kernels); + CoupledFlowAction(const Problem& problem, ExecutionSpace kernels); ~CoupledFlowAction(); CoupledFlowAction(const CoupledFlowAction&) = delete; CoupledFlowAction& operator=(const CoupledFlowAction&) = delete; diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense_output.hpp index 1143a7f16..381e0a6f6 100644 --- a/native/include/solprop_native/dense_output.hpp +++ b/native/include/solprop_native/dense_output.hpp @@ -4,15 +4,15 @@ #pragma once -#include -#include -#include - #include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dense.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include + /* * The native session's dense output over the resident state (solprop SEP 47 section * 5.3): the session-side half of `session::Dense`, whose field arithmetic is the diff --git a/native/include/solprop_native/dispatch.hpp b/native/include/solprop_native/dispatch.hpp new file mode 100644 index 000000000..71f2c7af8 --- /dev/null +++ b/native/include/solprop_native/dispatch.hpp @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include +#include + +namespace solprop_native { + +/** `function.template operator()()` for the first of `Values` equal to `value`, + else `otherwise()`: a run-time extent as a compile-time one, the `switch` ladder + written once. Device-callable; every branch returns the same type. */ +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) dispatch_among(const std::size_t value, + const Function& function, + const Otherwise& otherwise) { + if (value == First) + return function.template operator()(); + if constexpr (sizeof...(Rest) == 0) + return otherwise(); + else + return dispatch_among(value, function, otherwise); +} + +namespace detail { + +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) +dispatch_width(std::index_sequence /*offsets*/, const std::size_t width, + const Function& function, const Otherwise& otherwise) { + return dispatch_among<(First + Offsets)...>(width, function, otherwise); +} + +} // namespace detail + +/** `dispatch_among` over the widths `First, First + 1, ..., Last`. */ +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) dispatch_width(const std::size_t width, + const Function& function, + const Otherwise& otherwise) { + static_assert(First <= Last, "a width range holds at least one width"); + return detail::dispatch_width(std::make_index_sequence{}, + width, function, otherwise); +} + +} // namespace solprop_native diff --git a/native/include/solprop_native/dispersive.hpp b/native/include/solprop_native/dispersive.hpp index 614638c8b..a9c7cbedc 100644 --- a/native/include/solprop_native/dispersive.hpp +++ b/native/include/solprop_native/dispersive.hpp @@ -91,7 +91,12 @@ KOKKOS_INLINE_FUNCTION void dispersive_conserving(const Complex x0, const Comple const Complex y0, const Complex y1, const Complex z0, const Complex z1, Complex& out0, Complex& out1) { - Complex a0, a1, b0, b1, c0, c1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; + Complex c0; + Complex c1; dispersive_isotropic(x0, x1, y0, y1, Kokkos::conj(z0), Kokkos::conj(z1), a0, a1); dispersive_isotropic(x0, x1, Kokkos::conj(y0), Kokkos::conj(y1), z0, z1, b0, b1); dispersive_isotropic(Kokkos::conj(x0), Kokkos::conj(x1), y0, y1, z0, z1, c0, c1); @@ -105,11 +110,21 @@ KOKKOS_INLINE_FUNCTION void dispersive_harmonic(const Complex x0, const Complex const Complex z0, const Complex z1, const Complex carrier, Complex& out0, Complex& out1) { - Complex u0, u1, a0, a1, b0, b1, c0, c1; + Complex u0; + Complex u1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; + Complex c0; + Complex c1; dispersive_isotropic(x0, x1, y0, y1, z0, z1, u0, u1); - const Complex x0c = Kokkos::conj(x0), x1c = Kokkos::conj(x1); - const Complex y0c = Kokkos::conj(y0), y1c = Kokkos::conj(y1); - const Complex z0c = Kokkos::conj(z0), z1c = Kokkos::conj(z1); + const Complex x0c = Kokkos::conj(x0); + const Complex x1c = Kokkos::conj(x1); + const Complex y0c = Kokkos::conj(y0); + const Complex y1c = Kokkos::conj(y1); + const Complex z0c = Kokkos::conj(z0); + const Complex z1c = Kokkos::conj(z1); dispersive_isotropic(x0, x1, y0c, y1c, z0c, z1c, a0, a1); dispersive_isotropic(x0c, x1c, y0, y1, z0c, z1c, b0, b1); dispersive_isotropic(x0c, x1c, y0c, y1c, z0, z1, c0, c1); @@ -201,13 +216,17 @@ template KOKKOS_INLINE_FUNCTION void dispersive_conserving_n(const Complex (&x)[Components], const Complex (&y)[Components], const Complex (&z)[Components], Complex (&out)[Components]) { - Complex xc[Components], yc[Components], zc[Components]; + Complex xc[Components]; + Complex yc[Components]; + Complex zc[Components]; for (std::size_t c = 0; c < Components; ++c) { xc[c] = Kokkos::conj(x[c]); yc[c] = Kokkos::conj(y[c]); zc[c] = Kokkos::conj(z[c]); } - Complex a[Components], b[Components], d[Components]; + Complex a[Components]; + Complex b[Components]; + Complex d[Components]; dispersive_isotropic_n(x, y, zc, a); dispersive_isotropic_n(x, yc, z, b); dispersive_isotropic_n(xc, y, z, d); @@ -255,7 +274,7 @@ class DispersiveWeights { using RealView2 = Kokkos::View; using IndexView = Kokkos::View; - DispersiveWeights(const Problem& problem, const ExecutionSpace& kernels); + DispersiveWeights(const Problem& problem, ExecutionSpace kernels); ~DispersiveWeights() = default; DispersiveWeights(const DispersiveWeights&) = delete; DispersiveWeights& operator=(const DispersiveWeights&) = delete; @@ -423,10 +442,10 @@ class DispersiveResponse { bool borrowed_{}, shifted_{}, coarse_pending_{}, closed_{}; ExecutionSpace kernels_, response_, tail_response_, dense_, tail_dense_, matrix_copy_; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == response_chunk_ ? dense_ : tail_dense_; } StageViews stage_; @@ -536,10 +555,10 @@ class SeparableDispersive { std::size_t tail_rows_{}, active_rows_{}; bool active_tail_{}; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == response_chunk_ ? dense_ : tail_dense_; } StageViews stage_; @@ -627,7 +646,7 @@ class GeneralDispersive { std::size_t tail_rows_{}, active_rows_{}, last_chunk_{}; bool active_tail_{}; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } StageViews stage_; diff --git a/native/include/solprop_native/engines.hpp b/native/include/solprop_native/engines.hpp index dff00e333..13f23b52e 100644 --- a/native/include/solprop_native/engines.hpp +++ b/native/include/solprop_native/engines.hpp @@ -4,18 +4,18 @@ #pragma once -#include -#include -#include -#include -#include - #include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include +#include +#include + /* * The engines the native marches hand their field arithmetic to (solprop SEP 39 and * SEP 45): the resident engines over a native state, and the scripted engines of the diff --git a/native/include/solprop_native/envelope_plasma.hpp b/native/include/solprop_native/envelope_plasma.hpp index 5e0e58bb5..eb0ee415f 100644 --- a/native/include/solprop_native/envelope_plasma.hpp +++ b/native/include/solprop_native/envelope_plasma.hpp @@ -27,9 +27,12 @@ KOKKOS_INLINE_FUNCTION EnvelopeAxes envelope_axes( Kokkos::max(Kokkos::abs(second.real()), Kokkos::abs(second.imag()))); if (scale == 0.0) return {}; - const double xr = first.real() / scale, xi = first.imag() / scale; - const double yr = second.real() / scale, yi = second.imag() / scale; - const double xx = xr * xr + yr * yr, yy = xi * xi + yi * yi; + const double xr = first.real() / scale; + const double xi = first.imag() / scale; + const double yr = second.real() / scale; + const double yi = second.imag() / scale; + const double xx = xr * xr + yr * yr; + const double yy = xi * xi + yi * yi; const double xy = xr * xi + yr * yi; // A scaled norm avoids an underflow observed in the device hypot. const double larger = Kokkos::max(Kokkos::abs(xx - yy), Kokkos::abs(2 * xy)); @@ -40,20 +43,22 @@ KOKKOS_INLINE_FUNCTION EnvelopeAxes envelope_axes( const double determinant = xr * yi - xi * yr; const double ratio = Kokkos::min(1.0, Kokkos::abs(determinant) / major_square); const double angle = 0.5 * Kokkos::atan2(2 * xy, xx - yy); - const double cosine = Kokkos::cos(angle), sine = Kokkos::sin(angle); - const double ux = xr * cosine + xi * sine, uy = yr * cosine + yi * sine; + const double cosine = Kokkos::cos(angle); + const double sine = Kokkos::sin(angle); + const double ux = xr * cosine + xi * sine; + const double uy = yr * cosine + yi * sine; // The determinant retains the small axis near a degenerate linear ellipse. const double signed_ratio = determinant / major_square; - return {scale * Kokkos::sqrt(major_square), - ratio, - cosine, - sine, - ux, - uy, - -signed_ratio * uy, - signed_ratio * ux, - scale, - major_square}; + return {.major = scale * Kokkos::sqrt(major_square), + .ratio = ratio, + .cosine = cosine, + .sine = sine, + .ux = ux, + .uy = uy, + .vx = -signed_ratio * uy, + .vy = signed_ratio * ux, + .scale = scale, + .major_square = major_square}; } /** Positive directional cycle moments in s^-1 and a refinement failure flag. */ @@ -78,12 +83,13 @@ envelope_moments(const double major, const double ratio, const double field_min, return {}; if (ratio == 1.0) { const double rate = 0.5 * rate_at(major); - return {rate, rate, false}; + return {.cosine = rate, .sine = rate, .failed = false}; } if (!Kokkos::isfinite(major) || !Kokkos::isfinite(ratio)) - return {0.0, 0.0, true}; + return {.cosine = 0.0, .sine = 0.0, .failed = true}; constexpr double half_pi = std::numbers::pi_v / 2.0; - const double high = field_max / major, low = field_min / major; + const double high = field_max / major; + const double low = field_min / major; const double high_edge = high >= 1.0 ? 0.0 : high <= ratio ? half_pi @@ -94,10 +100,12 @@ envelope_moments(const double major, const double ratio, const double field_min, ? half_pi : Kokkos::acos(Kokkos::sqrt((low * low - ratio * ratio) / (1 - ratio * ratio))); - double previous_c = 0.0, previous_s = 0.0; + double previous_c = 0.0; + double previous_s = 0.0; for (std::size_t level = 0; level < 4; ++level) { const std::size_t order = std::size_t{64} << level; - double mc = 0.0, ms = 0.0; + double mc = 0.0; + double ms = 0.0; for (std::size_t panel = 0; panel < 3; ++panel) { const double lower = panel == 0 ? 0.0 : panel == 1 ? high_edge : low_edge; const double upper = panel == 0 ? high_edge : panel == 1 ? low_edge : half_pi; @@ -106,8 +114,10 @@ envelope_moments(const double major, const double ratio, const double field_min, continue; for (std::size_t index = 0; index < order; ++index) { const double theta = lower + (nodes(level, index) + 1) * (0.5 * width); - const double cosine = Kokkos::cos(theta), sine = Kokkos::sin(theta); - const double c2 = cosine * cosine, s2 = sine * sine; + const double cosine = Kokkos::cos(theta); + const double sine = Kokkos::sin(theta); + const double c2 = cosine * cosine; + const double s2 = sine * sine; const double denominator = c2 + ratio * ratio * s2; const double rate = rate_at(major * Kokkos::sqrt(denominator)) / denominator; const double weight = @@ -119,11 +129,11 @@ envelope_moments(const double major, const double ratio, const double field_min, if (level > 0 && Kokkos::isfinite(mc) && Kokkos::isfinite(ms) && Kokkos::abs(mc - previous_c) <= 2e-6 * Kokkos::max(mc, 1e-30) && Kokkos::abs(ms - previous_s) <= 2e-6 * Kokkos::max(ms, 1e-30)) - return {mc, ms, false}; + return {.cosine = mc, .sine = ms, .failed = false}; previous_c = mc; previous_s = ms; } - return {previous_c, previous_s, true}; + return {.cosine = previous_c, .sine = previous_s, .failed = true}; } /** Reconstruct the two loss-current components in A/m^2 from work in J/m^3. */ @@ -135,7 +145,8 @@ envelope_loss(const EnvelopeAxes& axes, const double mc, const double ms, const Kokkos::complex major_phase(axes.cosine, axes.sine); const Kokkos::complex minor_phase(-axes.sine, axes.cosine); // Dividing moments first avoids an overflowing standalone inverse tiny scale. - const double major_moment = mc / axes.scale, minor_moment = ms / axes.scale; + const double major_moment = mc / axes.scale; + const double minor_moment = ms / axes.scale; const double factor = 2 * work / axes.major_square; return {factor * (major_moment * axes.ux * major_phase + minor_moment * axes.vx * minor_phase), diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/far_field.hpp index 0e7b5b20d..061fec5e7 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/far_field.hpp @@ -147,7 +147,7 @@ KOKKOS_INLINE_FUNCTION std::size_t far_field_band_edge(const std::size_t band, * protected block of plain arithmetic rounds each operation on its own. CUDA's * intrinsics are real. */ -#if defined(__HIPCC__) +#ifdef __HIPCC__ #define SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT _Pragma("clang fp contract(off)") #else #define SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT @@ -172,7 +172,7 @@ KOKKOS_INLINE_FUNCTION double far_field_radius(const double scale, const double const double centre_y) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ const double across = __dadd_rn(__dmul_rn(scale, k_x), centre_x); const double along = __dadd_rn(__dmul_rn(scale, k_y), centre_y); return __dsqrt_rn(__dadd_rn(__dmul_rn(across, across), __dmul_rn(along, along))); @@ -196,7 +196,7 @@ KOKKOS_INLINE_FUNCTION double far_field_offset(const double scale, const double const double centre) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ return __dadd_rn(__dmul_rn(scale, k), centre); #else return scale * k + centre; @@ -216,7 +216,7 @@ KOKKOS_INLINE_FUNCTION double far_field_scaled_radius(const double across, const double stretch_y) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ const double scaled_x = __dmul_rn(stretch_x, across); const double scaled_y = __dmul_rn(stretch_y, along); return __dsqrt_rn( @@ -361,7 +361,7 @@ class FarFieldPass { double evanescent{}; }; - FarFieldPass(const Problem& problem, const ExecutionSpace& execution); + FarFieldPass(const Problem& problem, ExecutionSpace execution); ~FarFieldPass() = default; FarFieldPass(const FarFieldPass&) = delete; FarFieldPass& operator=(const FarFieldPass&) = delete; diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 6d173d3f1..8f9617209 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -4,19 +4,21 @@ #pragma once +#include "solprop_native/core.hpp" + #include -#include -#include "solprop_native/core.hpp" +#include +#include #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -#include +#include + #include #include #include #include #include -#include #include #include #include @@ -140,14 +142,13 @@ inline bool adds_wisdom(const std::string& planned, const std::string& stored) { found.emplace_back(text.data() + first, last - first); first = last + 1; } - std::sort(found.begin(), found.end()); + std::ranges::sort(found); return found; }; const auto had = lines(stored); - for (const auto line : lines(planned)) - if (!std::binary_search(had.begin(), had.end(), line)) - return true; - return false; + return std::ranges::any_of(lines(planned), [&had](const auto line) { + return !std::ranges::binary_search(had, line); + }); } // Merge `text` beneath the process's wisdom: where both hold an entry, the process's @@ -398,7 +399,7 @@ inline std::vector split_key(const std::string& text, const char by return parts; } -inline double median_execution(const fftw_plan plan) { +inline double median_execution(fftw_plan plan) { using Clock = std::chrono::steady_clock; fftw_execute(plan); const auto first = Clock::now(); @@ -412,7 +413,7 @@ inline double median_execution(const fftw_plan plan) { fftw_execute(plan); batch = std::chrono::duration(Clock::now() - start).count() / repeats; } - std::sort(batches.begin(), batches.end()); + std::ranges::sort(batches); return batches[2]; } } // namespace detail @@ -436,7 +437,8 @@ inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom // The step's waiting policy and FFTW team: without the scope LLVM's runtime sleeps // its idle threads at once (`KMP_BLOCKTIME=0`) and each loop pays their wake-up. const CpuTeamScope scope(threads); - const bool r2c = kind == "r2c", c2r = kind == "c2r"; + const bool r2c = kind == "r2c"; + const bool c2r = kind == "c2r"; const bool planes = kind.starts_with("planes-"); const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; if (!r2c && !c2r && !planes && !kind.starts_with("c2c-")) @@ -480,7 +482,7 @@ inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom if ((flags & FFTW_WISDOM_ONLY) != 0U && fftw_import_wisdom_from_string(wisdom.c_str()) == 0) return std::numeric_limits::quiet_NaN(); - const fftw_plan plan = make(flags); + auto* const plan = make(flags); if (plan == nullptr) { fftw_forget_wisdom(); return std::numeric_limits::quiet_NaN(); @@ -512,7 +514,6 @@ class FftPlanningScope { #include "solprop_native/graph.hpp" #include -#include #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) #include "solprop_native/launch.hpp" diff --git a/native/include/solprop_native/flow_functions.hpp b/native/include/solprop_native/flow_functions.hpp index 4a2aa8c91..d12f768b7 100644 --- a/native/include/solprop_native/flow_functions.hpp +++ b/native/include/solprop_native/flow_functions.hpp @@ -130,7 +130,7 @@ augmented_row(const Complex shift, const double length, const Triangular& triang // block and the next block row's scalars, a later block row the scalars alone. Each // new block is formed from the old ones into the scratch, then copied. for (int term = taylor_order; term > 0; --term) { - const double by = static_cast(term); + const auto by = static_cast(term); for (std::size_t j = 0; j <= count; ++j) { const double previous = j >= 1 ? p[j - 1] : 0.0; for (std::size_t row = 0; row < modes; ++row) diff --git a/native/include/solprop_native/fp.hpp b/native/include/solprop_native/fp.hpp index 1e0f8c3af..73d772512 100644 --- a/native/include/solprop_native/fp.hpp +++ b/native/include/solprop_native/fp.hpp @@ -10,7 +10,7 @@ * Floating-point contraction policy (solprop SEP 25, sections 4.2 and 5). * * CPU builds compile with contraction on across statements (`-ffp-contract=fast` for - * GCC, `-ffp-contract=fast-honor-pragmas` for Clang and icpx; CMakeLists.txt), so the + * GCC, `-ffp-contract=fast-honor-pragmas` for Clang; CMakeLists.txt), so the * compiler may fuse any multiply and add into one rounding. A protected block keeps * the arithmetic as written, each product and sum rounded on its own: the stable phase * increments, the small-argument phi-functions, the ionised fraction (every column's @@ -28,7 +28,7 @@ * function's declaration, after a lambda's parameter list) and starts its body with * `SOLPROP_NATIVE_STRICT_BLOCK`: * - * - Clang and icpx: `#pragma clang fp contract(off)` at the start of the body. The + * - Clang: `#pragma clang fp contract(off)` at the start of the body. The * setting is carried by each instruction, so it survives inlining. * - GCC ignores `#pragma STDC FP_CONTRACT` and Clang's pragma, so the function is * compiled with `optimize("fp-contract=off")`. GCC then does not inline it into a diff --git a/native/include/solprop_native/gemm.hpp b/native/include/solprop_native/gemm.hpp index 6b33b870e..8684fb0ec 100644 --- a/native/include/solprop_native/gemm.hpp +++ b/native/include/solprop_native/gemm.hpp @@ -17,35 +17,6 @@ namespace solprop_native { -enum class GemmProvider { custom, kokkosblas_native, kokkosblas_tpl, hybrid }; - -[[nodiscard]] constexpr GemmProvider compiled_gemm_provider() noexcept { -#if defined(SOLPROP_NATIVE_GEMM_PROVIDER_CUSTOM) - return GemmProvider::custom; -#elif defined(SOLPROP_NATIVE_GEMM_PROVIDER_KOKKOSBLAS_NATIVE) - return GemmProvider::kokkosblas_native; -#elif defined(SOLPROP_NATIVE_GEMM_PROVIDER_KOKKOSBLAS_TPL) - return GemmProvider::kokkosblas_tpl; -#else - return GemmProvider::hybrid; -#endif -} - -[[nodiscard]] constexpr std::string_view -gemm_provider_name(const GemmProvider provider) noexcept { - switch (provider) { - case GemmProvider::custom: - return "custom"; - case GemmProvider::kokkosblas_native: - return "kokkosblas-native"; - case GemmProvider::kokkosblas_tpl: - return "kokkosblas-tpl"; - case GemmProvider::hybrid: - return "hybrid"; - } - return "unknown"; -} - // A host build without a vendor BLAS runs KokkosBlas's native real GEMM arithmetic // through `native_order_gemm`: the same result to // the bit, several times faster on the engine's skinny-by-wide transforms. @@ -205,7 +176,7 @@ void native_order_gemm(const ExecutionSpace& execution_space, const AView& a, // at 316 GFLOP/s, about 54 % of the FP64 peak, at any row pitch. Such a build forms // the engine's real GEMMs with `device_dense_gemm`. inline constexpr bool device_dense_gemm_enabled = -#if defined(KOKKOS_ARCH_AMPERE86) +#ifdef KOKKOS_ARCH_AMPERE86 true; #else false; @@ -360,9 +331,9 @@ void selected_gemm(const ExecutionSpace& execution_space, const AView& a, const BView& b, const CView& c, const typename CView::non_const_value_type beta, const bool tpl_eligible) { - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom || - (provider == GemmProvider::hybrid && !tpl_eligible)) { + // The hybrid provider: the engine's GEMM for an eligible transform, the custom + // kernel for the rest. + if (!tpl_eligible) { custom_gemm(execution_space, a, b, c, beta); return; } diff --git a/native/include/solprop_native/general_modal.hpp b/native/include/solprop_native/general_modal.hpp index e0ccd8ee6..4f8fafc5f 100644 --- a/native/include/solprop_native/general_modal.hpp +++ b/native/include/solprop_native/general_modal.hpp @@ -4,6 +4,7 @@ #pragma once +#include "solprop_native/build.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dispersive.hpp" #include "solprop_native/raman_fft.hpp" @@ -67,11 +68,7 @@ constexpr std::size_t general_projection_lanes(const std::size_t modes, * (exactly, a power of two): half the inverse transform's work and half the field's * memory traffic. Host builds only; device builds keep the complex transform. */ -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -inline constexpr bool general_real_signal_enabled = false; -#else -inline constexpr bool general_real_signal_enabled = true; -#endif +inline constexpr bool general_real_signal_enabled = build::host; /** * Refuse a general modal problem whose extents or data this source cannot execute. @@ -230,10 +227,10 @@ class GeneralModalSource { CpuExecutionPool kernel_pool_, fft_pool_, raman_pool_, raman_tail_pool_; ExecutionSpace kernels_, transforms_, delayed_transforms_; ExecutionSpace response_, tail_response_, dense_, tail_dense_, maps_; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == chunk_ ? dense_ : tail_dense_; } ComplexView synthesis_, projection_, frequency_factor_, delayed_spectrum_; diff --git a/native/include/solprop_native/graph.hpp b/native/include/solprop_native/graph.hpp index 0b4752696..3e9b1e040 100644 --- a/native/include/solprop_native/graph.hpp +++ b/native/include/solprop_native/graph.hpp @@ -729,8 +729,9 @@ namespace detail { /** Whether a view of `Memory` lives where `Space` runs and not on the host. */ template inline constexpr bool device_resident = - Kokkos::SpaceAccessibility::accessible && - !Kokkos::SpaceAccessibility::accessible; + static_cast(Kokkos::SpaceAccessibility::accessible) && + !static_cast( + Kokkos::SpaceAccessibility::accessible); /** A copy or fill between device buffers, whose kernel (if Kokkos needs one) passes its views as its argument. */ diff --git a/native/include/solprop_native/lagrange_basis.hpp b/native/include/solprop_native/lagrange_basis.hpp index bf11a26b8..67b73bb7e 100644 --- a/native/include/solprop_native/lagrange_basis.hpp +++ b/native/include/solprop_native/lagrange_basis.hpp @@ -164,10 +164,12 @@ inline void trial_tables(const double* nodes, const std::size_t order, for (std::size_t t = 0; t + 1 < width; ++t) spans[t] = 1.0 - nodes[t]; basis_rows(nodes, order, 1.0, true, scratch, carried); + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide basis_rows(through, order + 1, 1.0, true, scratch, endpoint); for (std::size_t index = 0; index < n_orders; ++index) { const std::size_t scored = orders[index]; double* const row = defects + index * width; + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide basis_rows(through, scored + 1, 1.0, true, scratch, row); for (std::size_t t = scored + 1; t < width; ++t) row[t] = 0.0; diff --git a/native/include/solprop_native/launch.hpp b/native/include/solprop_native/launch.hpp index 70f15c7ca..7d2ae3ab4 100644 --- a/native/include/solprop_native/launch.hpp +++ b/native/include/solprop_native/launch.hpp @@ -4,10 +4,10 @@ #pragma once -#include - #include "solprop_native/graph.hpp" +#include + #include #include #include @@ -69,7 +69,7 @@ void parallel_for(const Label& label, const Policy& policy, const Functor& funct left three of the device's compute units to do the work. */ template [[nodiscard]] auto reduce_policy_for(const Policy& policy) { -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP if constexpr (Kokkos::is_execution_policy_v) { using Space = typename Policy::execution_space; if constexpr (std::is_same_v> && diff --git a/native/include/solprop_native/low_storage.hpp b/native/include/solprop_native/low_storage.hpp index c661abe3f..7787dff57 100644 --- a/native/include/solprop_native/low_storage.hpp +++ b/native/include/solprop_native/low_storage.hpp @@ -4,15 +4,15 @@ #pragma once -#include -#include -#include - #include "solprop_native/core.hpp" #include "solprop_native/engines.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include + /* * The low-storage RK4IP replay in the native session (solprop SEP 24, SEP 47 section * 6.3): the engine that takes one in-place step of the march from its stage scalars, diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/plasma.hpp index ef707d1aa..ff24af419 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/plasma.hpp @@ -5,6 +5,7 @@ #pragma once #include "solprop_native/core.hpp" +#include "solprop_native/dispatch.hpp" #include @@ -215,26 +216,12 @@ KOKKOS_INLINE_FUNCTION void band_filter_interior(const double* row, double* out, KOKKOS_INLINE_FUNCTION void band_filter_interior(const double* row, double* out, const int low, const int high, const double* taps, const int half) { - switch (half) { - case 0: - return band_filter_interior<0>(row, out, low, high, taps); - case 1: - return band_filter_interior<1>(row, out, low, high, taps); - case 2: - return band_filter_interior<2>(row, out, low, high, taps); - case 3: - return band_filter_interior<3>(row, out, low, high, taps); - case 4: - return band_filter_interior<4>(row, out, low, high, taps); - case 5: - return band_filter_interior<5>(row, out, low, high, taps); - case 6: - return band_filter_interior<6>(row, out, low, high, taps); - case 7: - return band_filter_interior<7>(row, out, low, high, taps); - default: - return band_filter_interior(row, out, low, high, taps); - } + const auto filter = [&] { + band_filter_interior(Width)>(row, out, low, high, taps); + }; + dispatch_width<0, band_max_half - 1>(static_cast(half), filter, [&] { + band_filter_interior(row, out, low, high, taps); + }); } /** A staged band filter's tile in a team's scratch. */ @@ -320,9 +307,9 @@ KOKKOS_INLINE_FUNCTION BandWindow band_support(const int live_first, const int live_last, const int samples, const int half) { if (live_first > live_last) - return {samples, -1}; - return {live_first > half ? live_first - half : 0, - live_last + half < samples - 1 ? live_last + half : samples - 1}; + return {.first = samples, .last = -1}; + return {.first = live_first > half ? live_first - half : 0, + .last = live_last + half < samples - 1 ? live_last + half : samples - 1}; } /** diff --git a/native/include/solprop_native/raman_fft.hpp b/native/include/solprop_native/raman_fft.hpp index 336d7ad0c..515c28468 100644 --- a/native/include/solprop_native/raman_fft.hpp +++ b/native/include/solprop_native/raman_fft.hpp @@ -19,7 +19,7 @@ namespace solprop_native { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE // The host adapter owns only vendor setup data. All field storage remains in the // workspace's Kokkos Views and is included in its existing allocation ledger. class AccelerateRamanPlan { @@ -30,6 +30,8 @@ class AccelerateRamanPlan { ~AccelerateRamanPlan(); AccelerateRamanPlan(const AccelerateRamanPlan&) = delete; AccelerateRamanPlan& operator=(const AccelerateRamanPlan&) = delete; + AccelerateRamanPlan(AccelerateRamanPlan&&) = delete; + AccelerateRamanPlan& operator=(AccelerateRamanPlan&&) = delete; void execute_task(std::size_t task) const; [[nodiscard]] int tasks() const noexcept; @@ -49,7 +51,7 @@ template class RamanFftPlan { #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) const bool contiguous = (axis == -1 || axis == 1) && input.stride(1) == 1 && output.stride(1) == 1; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE constexpr bool forward = std::is_same_v; const auto samples = forward ? input.extent(1) : output.extent(1); @@ -91,7 +93,7 @@ template class RamanFftPlan { void execute(const Input& input, const Output& output, const KokkosFFT::Normalization normalization) const { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) { if (normalization != KokkosFFT::Normalization::none) throw std::invalid_argument("Raman vDSP requires explicit normalization"); @@ -114,7 +116,7 @@ template class RamanFftPlan { } [[nodiscard]] const char* provider() const noexcept { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) return "Accelerate_vDSP"; #endif @@ -122,7 +124,7 @@ template class RamanFftPlan { } [[nodiscard]] int tasks(const int fallback_threads) const noexcept { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) return accelerate_->tasks(); #endif @@ -133,7 +135,7 @@ template class RamanFftPlan { using Fallback = KokkosFFT::Plan; Execution execution_; std::unique_ptr fallback_; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE std::unique_ptr accelerate_; #endif }; diff --git a/native/include/solprop_native/refuse.hpp b/native/include/solprop_native/refuse.hpp new file mode 100644 index 000000000..ab8e85707 --- /dev/null +++ b/native/include/solprop_native/refuse.hpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include + +namespace solprop_native { + +namespace detail { + +template void append_part(std::string& text, const Part& part) { + if constexpr (std::is_arithmetic_v && !std::is_same_v && + !std::is_same_v) + text += std::to_string(part); + else + text += std::string_view(part); +} + +} // namespace detail + +/** `parts` concatenated: text as given, numbers as `std::to_string` writes them (a + double in `%f`), so that a message built from them keeps its exact text. */ +template [[nodiscard]] std::string concat(const Parts&... parts) { + std::string text; + (detail::append_part(text, parts), ...); + return text; +} + +/** Throw `Error` with the message `concat(parts...)`. */ +template +[[noreturn]] void refuse(const Parts&... parts) { + throw Error(concat(parts...)); +} + +} // namespace solprop_native diff --git a/native/include/solprop_native/rk_march.hpp b/native/include/solprop_native/rk_march.hpp index 4c35dc2ba..6d8646915 100644 --- a/native/include/solprop_native/rk_march.hpp +++ b/native/include/solprop_native/rk_march.hpp @@ -4,14 +4,14 @@ #pragma once +#include "solprop_native/adams_march.hpp" + #include #include #include #include #include -#include "solprop_native/adams_march.hpp" - /* * The Runge-Kutta march (solprop SEP 45): a trial's stage positions and its scaled * error norm (phase 2), and the adaptive march itself, its controller and its loop diff --git a/native/include/solprop_native/session.hpp b/native/include/solprop_native/session.hpp index babe8fdf4..34632b63a 100644 --- a/native/include/solprop_native/session.hpp +++ b/native/include/solprop_native/session.hpp @@ -4,6 +4,13 @@ #pragma once +#include "solprop_native/adams_march.hpp" +#include "solprop_native/dense.hpp" +#include "solprop_native/rk_march.hpp" +#include "solprop_native/statistics_plan.hpp" +#include "solprop_native/transverse.hpp" +#include "solprop_native/watch.hpp" + #include #include #include @@ -20,13 +27,6 @@ #include #include -#include "solprop_native/adams_march.hpp" -#include "solprop_native/dense.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/statistics_plan.hpp" -#include "solprop_native/transverse.hpp" -#include "solprop_native/watch.hpp" - /* * The native execution session (solprop SEP 47 sections 5 and 6.1): the one mutable * owner of a trajectory. It holds the march's policy, state and history, the plane diff --git a/native/include/solprop_native/stage_tables.hpp b/native/include/solprop_native/stage_tables.hpp index 263f44a39..209cb6008 100644 --- a/native/include/solprop_native/stage_tables.hpp +++ b/native/include/solprop_native/stage_tables.hpp @@ -7,9 +7,11 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" #include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/refuse.hpp" #include #include +#include #include #include #include @@ -56,12 +58,13 @@ struct Table { chebyshev::clenshaw_points(&z, 1, edges.data(), edges.size(), lower, upper, coefficients.data(), n_terms, n_values, out); if (bad >= 0) - throw std::invalid_argument("z = " + std::to_string(z) + - " is outside a stage table's span"); + refuse("z = ", z, " is outside a stage table's span"); } - /** The table's single value at `z`. */ + /** The table's single value at `z`. The scalar tables are checked to hold one value + each when they are loaded (`require_scalar_tables`). */ [[nodiscard]] double value(const double z) const { + assert(n_values <= 1); double out = 0.0; at(z, &out); return out; @@ -74,8 +77,7 @@ struct Table { */ void closing(const double z, double* out, const bool before = true) const { if (z < lower || z > upper) - throw std::invalid_argument("z = " + std::to_string(z) + - " is outside a stage table's span"); + refuse("z = ", z, " is outside a stage table's span"); std::size_t index = chebyshev::interval_index(z, edges.data(), edges.size()); if (before && index > 0 && z <= edges[index]) --index; @@ -159,7 +161,7 @@ struct Sections { private: [[nodiscard]] std::size_t clamped(const double key) const { // `numpy.searchsorted(edges, key, side="right") - 1`, clamped to the sections. - const auto above = std::upper_bound(edges.begin(), edges.end(), key); + const auto above = std::ranges::upper_bound(edges, key); const auto raw = static_cast(above - edges.begin()) - 1; const auto last = static_cast(edges.size()) - 2; return static_cast(std::min(std::max(raw, std::ptrdiff_t{0}), last)); @@ -254,17 +256,15 @@ struct ZSlots { const std::vector& positions) { std::vector wanted; for (const double z : positions) - if (std::find(wanted.begin(), wanted.end(), z) == wanted.end()) + if (std::ranges::find(wanted, z) == wanted.end()) wanted.push_back(z); if (wanted.size() > slots) - throw std::invalid_argument("a native call reads the z table at " + - std::to_string(wanted.size()) + - " positions, more than the " + std::to_string(slots) + - " slots this problem holds"); + refuse("a native call reads the z table at ", wanted.size(), + " positions, more than the ", slots, " slots this problem holds"); std::vector slot_of(wanted.size(), slots); std::vector taken(slots, false); for (std::size_t index = 0; index < wanted.size(); ++index) { - const auto at = std::find(held.begin(), held.end(), wanted[index]); + const auto at = std::ranges::find(held, wanted[index]); if (at != held.end()) { slot_of[index] = static_cast(at - held.begin()); taken[slot_of[index]] = true; @@ -296,8 +296,8 @@ struct ZSlots { std::vector out; out.reserve(positions.size()); for (const double z : positions) - out.push_back(slot_of[static_cast( - std::find(wanted.begin(), wanted.end(), z) - wanted.begin())]); + out.push_back(slot_of[static_cast(std::ranges::find(wanted, z) - + wanted.begin())]); return out; } @@ -314,7 +314,7 @@ struct ZSlots { private: void touch(const std::size_t slot) { - order.erase(std::find(order.begin(), order.end(), slot)); + order.erase(std::ranges::find(order, slot)); order.push_back(slot); } @@ -398,8 +398,7 @@ inline Row row(const Tables& tables, Anchor& anchor, const double z0, const doub out.prefactor[at_slot] = 1.0; } else if (tables.prefactor == Terms::constant) { if (!(tables.prefactor_low <= z1 && z1 <= tables.prefactor_high)) - throw std::invalid_argument("z = " + std::to_string(z1) + - " lies outside the prefactor's table"); + refuse("z = ", z1, " lies outside the prefactor's table"); out.prefactor = {1.0}; } else { out.prefactor.resize(tables.prefactor_weights.n_values); @@ -540,7 +539,7 @@ inline TrialScalars sample(const Tables& tables, Anchor& anchor, if (tables.flow.present()) return trial(tables, row(tables, anchor, end, end)); auto out = step; - std::fill(out.phi_delta.begin(), out.phi_delta.end(), 0.0); + std::ranges::fill(out.phi_delta, 0.0); return out; } @@ -554,6 +553,34 @@ inline TrialScalars sample(const Tables& tables, Anchor& anchor, * real position forms the imaginary part by these operations, so the two agree but for * the sign of a zero (solprop issue #255). */ +/** + * Refuse, by name, a scalar stage table that holds other than one value: the density, + * taper, Kerr density, Raman rates and densities, plasma densities and preionisation, + * dispersive densities and the chord's table, each read with `Table::value`. An absent + * optional table (no values) is not checked. + */ +inline void require_scalar_tables(const Tables& tables) { + const auto scalar = [](const Table& table, const std::string& name) { + if (table.present() && table.n_values != 1) + refuse("the stage table ", name, " holds ", table.n_values, + " values where one is read"); + }; + const auto scalars = [&scalar](const std::vector& list, + const std::string& name) { + for (std::size_t index = 0; index < list.size(); ++index) + scalar(list[index], concat(name, "[", index, "]")); + }; + scalar(tables.density, "density"); + scalar(tables.taper, "taper"); + scalar(tables.kerr_density, "kerr_density"); + scalar(tables.chord_table, "chord_table"); + scalars(tables.raman_rates, "raman_rates"); + scalars(tables.plasma_densities, "plasma_densities"); + scalars(tables.plasma_preionisation, "plasma_preionisation"); + scalars(tables.raman_densities, "raman_densities"); + scalars(tables.dispersive_densities, "dispersive_densities"); +} + inline double chord(const Tables& tables, const double z1, const double z2) { if (tables.chord == Chord::constant) return tables.chord_rate * (z2 - z1); @@ -623,7 +650,7 @@ inline ExponentialWeights weights(const double* nodes, const std::size_t n, const std::size_t width = highest + 1; std::array predictor{}; std::array corrector{}; - std::array factored{}; + std::array factored{}; std::array spans{}; std::array carried{}; std::array endpoint{}; diff --git a/native/include/solprop_native/vmath.hpp b/native/include/solprop_native/vmath.hpp index 6d5cc4aea..752774246 100644 --- a/native/include/solprop_native/vmath.hpp +++ b/native/include/solprop_native/vmath.hpp @@ -7,10 +7,10 @@ #include #include -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) -#include - +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF #include + +#include #endif /* @@ -137,7 +137,7 @@ struct Lanes { #error "solprop-native: SLEEF vector maths needs an x86-64 or aarch64 target" #endif -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF inline constexpr bool vectorised = true; inline constexpr std::size_t width = Lanes::width; // The variant, for the build record. diff --git a/native/include/solprop_native/z_table.hpp b/native/include/solprop_native/z_table.hpp index d0082b564..e429e09c0 100644 --- a/native/include/solprop_native/z_table.hpp +++ b/native/include/solprop_native/z_table.hpp @@ -6,6 +6,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -68,9 +69,8 @@ inline Points locate(const Table& table, const std::vector& slots, if (slots.size() != positions.size()) throw std::invalid_argument("z table: one slot per position"); if (positions.size() > max_points) - throw std::invalid_argument("z table: at most " + std::to_string(max_points) + - " positions per evaluation, got " + - std::to_string(positions.size())); + refuse("z table: at most ", max_points, " positions per evaluation, got ", + positions.size()); if (table.edges.size() < 2) throw std::logic_error("z table: no table has been uploaded"); Points points; @@ -78,10 +78,8 @@ inline Points locate(const Table& table, const std::vector& slots, for (std::size_t p = 0; p < positions.size(); ++p) { const double z = positions[p]; if (!(z >= table.lower && z <= table.upper)) - throw std::out_of_range("z = " + std::to_string(z) + - " m is outside the z table's span [" + - std::to_string(table.edges.front()) + ", " + - std::to_string(table.edges.back()) + "] m"); + refuse("z = ", z, " m is outside the z table's span [", + table.edges.front(), ", ", table.edges.back(), "] m"); std::size_t k = chebyshev::interval_index(z, table.edges.data(), table.edges.size()); if (before && k > 0 && z <= table.edges[k]) diff --git a/native/pyproject.toml b/native/pyproject.toml index a0a634f4c..cc632fa2e 100644 --- a/native/pyproject.toml +++ b/native/pyproject.toml @@ -114,7 +114,7 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} --requir # copying libomp.dll into the wheel under a mangled name. [tool.cibuildwheel.windows] before-all = "pwsh -NoProfile -ExecutionPolicy Bypass -File {package}\\ci\\build-deps-windows.ps1 -Prefix C:\\solprop-deps" -environment = { CMAKE_GENERATOR = "Ninja", CC = "clang", CXX = "clang++", CMAKE_ARGS = "-DSOLPROP_NATIVE_DISTRIBUTION_BLAS=OPENMP_OPENBLAS -DSOLPROP_NATIVE_OPENBLAS_ROOT=C:/solprop-deps/openblas -DFFTW_ROOT=C:/solprop-deps/fftw -DFFTW_DOUBLE_LIB=C:/solprop-deps/fftw/lib/fftw3.lib -DFFTW_FLOAT_LIB=C:/solprop-deps/fftw/lib/fftw3f.lib -DFFTW_DOUBLE_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3_omp.lib -DFFTW_FLOAT_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3f_omp.lib -DOpenMP_CXX_FLAGS=-fopenmp -DOpenMP_CXX_LIB_NAMES=libomp -DOpenMP_libomp_LIBRARY=C:/solprop-deps/libomp/lib/libomp.lib -DSOLPROP_NATIVE_BUNDLED_LICENCES=C:/solprop-deps/licences" } +environment = { CMAKE_GENERATOR = "Ninja", CC = "clang", CXX = "clang++", CMAKE_ARGS = "-DSOLPROP_NATIVE_DISTRIBUTION_BLAS=OPENMP_OPENBLAS -DSOLPROP_NATIVE_OPENBLAS_ROOT=C:/solprop-deps/openblas -DFFTW_ROOT=C:/solprop-deps/fftw -DFFTW_DOUBLE_LIB=C:/solprop-deps/fftw/lib/fftw3.lib -DFFTW_DOUBLE_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3_omp.lib -DOpenMP_CXX_FLAGS=-fopenmp -DOpenMP_CXX_LIB_NAMES=libomp -DOpenMP_libomp_LIBRARY=C:/solprop-deps/libomp/lib/libomp.lib -DSOLPROP_NATIVE_BUNDLED_LICENCES=C:/solprop-deps/licences" } repair-wheel-command = "delvewheel repair --add-path C:\\solprop-deps\\libomp\\bin -w {dest_dir} -v {wheel}" # The x86-64 wheels hold two builds of the extension: x86-64-v3, and x86-64-v2 for older @@ -160,7 +160,7 @@ select = ["B", "E", "F", "I", "PERF", "PT", "RUF", "UP", "W"] "tests/**" = ["S101"] [tool.pyright] -include = ["src", "tests", "benchmarks"] +include = ["src", "tests"] exclude = ["build", "dist", "dist-*", ".venv"] extraPaths = ["src", "../src"] pythonVersion = "3.14" diff --git a/native/scripts/check.sh b/native/scripts/check.sh index 73da7067a..1361f8975 100755 --- a/native/scripts/check.sh +++ b/native/scripts/check.sh @@ -13,65 +13,16 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) workspace_root=$(cd "$repo_root/.." && pwd) cd "$repo_root" -raman_default=KOKKOSFFT -if [[ $(uname -s) == Darwin ]]; then - raman_default=ACCELERATE -fi -raman_provider=${SOLPROP_NATIVE_CHECK_RAMAN_FFT_PROVIDER:-$raman_default} +# shellcheck source=native/scripts/compiler.sh +source "$repo_root/scripts/compiler.sh" +raman_provider=${SOLPROP_NATIVE_CHECK_RAMAN_FFT_PROVIDER:-$(native_raman_provider)} export CMAKE_ARGS="${CMAKE_ARGS:-} -DSOLPROP_NATIVE_RAMAN_FFT_PROVIDER=$raman_provider" - -if [[ $(uname -s) == "Linux" ]]; then - # CXX when set (Linux CPU builds need GCC 14 or newer; Ubuntu 24.04's g++ is 13). - cxx_compiler=$(command -v "${CXX:-g++}") -elif [[ -x /opt/homebrew/opt/llvm/bin/clang++ ]]; then - cxx_compiler=/opt/homebrew/opt/llvm/bin/clang++ -elif [[ -x /usr/local/opt/llvm/bin/clang++ ]]; then - cxx_compiler=/usr/local/opt/llvm/bin/clang++ -else - cxx_compiler=$(command -v clang++) -fi -cpp_sources=( - benchmarks/cpu_timer.cpp - include/solprop_native/chebyshev.hpp - include/solprop_native/core.hpp - include/solprop_native/coupled_flow.hpp - include/solprop_native/dispersive.hpp - include/solprop_native/envelope_plasma.hpp - include/solprop_native/far_field.hpp - include/solprop_native/fft_plan.hpp - include/solprop_native/fp.hpp - include/solprop_native/general_modal.hpp - include/solprop_native/raman_fft.hpp - include/solprop_native/gemm.hpp - include/solprop_native/lagrange_basis.hpp - include/solprop_native/plasma.hpp - include/solprop_native/slab.hpp - include/solprop_native/z_table.hpp - src/bindings.cpp - src/coverage_labels.cpp - src/core.cpp - src/coupled_flow.cpp - src/dispersive.cpp - src/far_field.cpp - src/general.cpp - src/general_modal.cpp - src/helpers.cpp - src/helpers.hpp - src/raman_fft.cpp - src/benchmark_gemm.cpp - tests/native/bounds.hpp - tests/native/main.cpp - tests/native/test_aperture.cpp - tests/native/test_core.cpp - tests/native/test_coupled_flow.cpp - tests/native/test_dispersive.cpp - tests/native/test_envelope_plasma.cpp - tests/native/test_fft.cpp - tests/native/test_fp.cpp - tests/native/test_general_modal.cpp - tests/native/test_helpers.cpp - tests/native/test_z_table.cpp -) +cxx_compiler=$(native_cxx_compiler) +# Every tracked C++ file, so that none can drop out of the format check. +cpp_sources=() +while IFS= read -r -d '' cpp_source; do + cpp_sources+=("$cpp_source") +done < <(git ls-files -z -- '*.cpp' '*.hpp' '*.h') # The stages to run, all of them by default; CI runs them as parallel jobs. "lint" is # ruff, pyright and clang-format; "tidy" and "sanitize" build and run the native tests diff --git a/native/scripts/compiler.sh b/native/scripts/compiler.sh new file mode 100644 index 000000000..df0c9f3c1 --- /dev/null +++ b/native/scripts/compiler.sh @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Sourced by check.sh and tidy.sh: the C++ compiler and Raman FFT provider of the native +# gate's builds, chosen in one place. + +# The compiler: CXX on Linux (CPU builds need GCC 14 or newer; Ubuntu 24.04's g++ is 13), +# Homebrew's Clang on macOS (Apple silicon, then Intel), else the clang++ on PATH. +native_cxx_compiler() { + if [[ $(uname -s) == Linux ]]; then + command -v "${CXX:-g++}" + elif [[ -x /opt/homebrew/opt/llvm/bin/clang++ ]]; then + echo /opt/homebrew/opt/llvm/bin/clang++ + elif [[ -x /usr/local/opt/llvm/bin/clang++ ]]; then + echo /usr/local/opt/llvm/bin/clang++ + else + command -v clang++ + fi +} + +# The Raman FFT provider: Accelerate on macOS, Kokkos-FFT elsewhere. +native_raman_provider() { + if [[ $(uname -s) == Darwin ]]; then + echo ACCELERATE + else + echo KOKKOSFFT + fi +} diff --git a/native/scripts/tidy.sh b/native/scripts/tidy.sh new file mode 100755 index 000000000..ac4b99a92 --- /dev/null +++ b/native/scripts/tidy.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Run the pinned clang-tidy over some native C++ files (`poe tidy-native [FILE...]` from +# the repository root), with the compile commands of the cpu-tidy preset, which this +# configures when they are missing or stale, and the clang-tidy command it resolved. With no files it checks the C++ files under native/ +# that differ from main (committed, staged or not) and the untracked ones. Paths are +# relative to the repository root or absolute. The full check, every file as it +# compiles, is the gate's tidy stage. + +set -euo pipefail + +native=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +root=$(cd "$native/.." && pwd) +build="$native/build/presets/cpu-tidy" +cd "$root" + +files=() +if [[ $# -gt 0 ]]; then + files=("$@") +else + base=$(git merge-base "${SOLPROP_TIDY_BASE:-origin/main}" HEAD) + while IFS= read -r -d '' file; do + [[ -f $file ]] && files+=("$file") + done < <({ git diff -z --name-only "$base" -- 'native/*.cpp' 'native/*.hpp' 'native/*.h' + git ls-files -z --others --exclude-standard -- 'native/*.cpp' 'native/*.hpp' 'native/*.h'; } | + sort -zu) +fi +if [[ ${#files[@]} -eq 0 ]]; then + echo "tidy-native: no C++ files to check" + exit 0 +fi + +# shellcheck source=native/scripts/compiler.sh +source "$native/scripts/compiler.sh" +command_file="$build/clang_tidy_command.txt" +# Configure the cpu-tidy preset when its compile commands or clang-tidy command are +# missing or older than the build configuration. +if [[ ! -f "$build/compile_commands.json" || ! -f "$command_file" || + "$native/CMakeLists.txt" -nt "$build/compile_commands.json" || + "$native/CMakeLists.txt" -nt "$command_file" ]]; then + mkdir -p "$native/build" + log="$native/build/tidy-configure.log" + echo "tidy-native: configuring the cpu-tidy preset (log: $log)" + if ! (cd "$native" && cmake --preset cpu-tidy -DCMAKE_CXX_COMPILER="$(native_cxx_compiler)" \ + -DSOLPROP_NATIVE_RAMAN_FFT_PROVIDER="$(native_raman_provider)" \ + -DPython_EXECUTABLE="$root/.venv/bin/python") >"$log" 2>&1; then + tail -n 30 "$log" >&2 + exit 1 + fi +fi + +# SLEEF's header is generated by its own build, which a configure alone does not run. +if [[ -d "$build/_sleef" && ! -f "$build/_sleef/include/sleef.h" ]]; then + cmake --build "$build" --target solprop_native_sleef_build >"$native/build/tidy-sleef.log" 2>&1 +fi + +# The command the build runs: the clang-tidy CMake resolved, its configuration and +# header filter, and the parser arguments beyond the compile commands. +tidy_command=() +while IFS= read -r argument; do + [[ -n $argument ]] && tidy_command+=("$argument") +done < "$command_file" +pinned=$(sed -n 's/.*"clang-tidy==\([0-9.]*\).*/\1/p' "$root/pyproject.toml") +found=$("${tidy_command[0]}" --version | sed -n 's/.*LLVM version \([0-9.]*\).*/\1/p') +if [[ -z $pinned || $found != "$pinned" ]]; then + echo "tidy-native: ${tidy_command[0]} is clang-tidy ${found:-of unknown version}, not the" \ + "pinned ${pinned:-?}; run uv sync, then remove $build to reconfigure" >&2 + exit 1 +fi +# A header is checked with the commands of a source file near it (clang-tidy +# interpolates them), and through the sources that include it. +printf '%s\0' "${files[@]}" | + xargs -0 -n 1 -P "$(getconf _NPROCESSORS_ONLN)" "${tidy_command[@]}" --quiet -p "$build" diff --git a/native/src/adams_march.cpp b/native/src/adams_march.cpp index 2c9b0b7b2..8a32a61a8 100644 --- a/native/src/adams_march.cpp +++ b/native/src/adams_march.cpp @@ -4,6 +4,9 @@ #include "solprop_native/adams_march.hpp" +#include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" + #include #include #include @@ -12,8 +15,6 @@ #include #include -#include "solprop_native/core.hpp" - // GCC does not take a file's `-ffp-contract=off` as an inlining barrier: under // link-time optimisation, with the library's symbols hidden, it would inline a function // of this file into a contracting caller and fuse it (solprop #135). The optimize @@ -48,27 +49,24 @@ double py_max(const double a, const double b) { return b > a ? b : a; } /** `bisect.bisect_left` over an increasing sequence. */ std::size_t bisect_left(const std::vector& values, const double x) { - return static_cast(std::lower_bound(values.begin(), values.end(), x) - - values.begin()); + return static_cast(std::ranges::lower_bound(values, x) - values.begin()); } /** `bisect.bisect_right` over an increasing sequence. */ std::size_t bisect_right(const std::vector& values, const double x) { - return static_cast(std::upper_bound(values.begin(), values.end(), x) - - values.begin()); + return static_cast(std::ranges::upper_bound(values, x) - values.begin()); } -void refuse(const char* field, const double value) { - throw std::invalid_argument(std::string("the Adams march's policy has ") + field + - " = " + std::to_string(value)); +[[noreturn]] void refuse_policy(const char* field, const double value) { + refuse("the Adams march's policy has ", field, " = ", value); } bool increasing(const std::vector& values) { for (std::size_t index = 1; index < values.size(); ++index) if (!(values[index] > values[index - 1])) return false; - return std::all_of(values.begin(), values.end(), - [](const double value) { return std::isfinite(value); }); + return std::ranges::all_of(values, + [](const double value) { return std::isfinite(value); }); } } // namespace @@ -87,46 +85,46 @@ double strict_probe(const double a, const double b, const double c) { void validate(const Policy& policy) { if (policy.order < 1 || policy.order > 6) - refuse("order", static_cast(policy.order)); + refuse_policy("order", static_cast(policy.order)); if (policy.real_intervals.size() < policy.order) - refuse("real_intervals", static_cast(policy.real_intervals.size())); + refuse_policy("real_intervals", static_cast(policy.real_intervals.size())); if (!(policy.safety > 0.0 && policy.safety <= 1.0)) - refuse("safety", policy.safety); + refuse_policy("safety", policy.safety); if (!(policy.max_growth > 1.0)) - refuse("max_growth", policy.max_growth); + refuse_policy("max_growth", policy.max_growth); if (!(policy.min_factor > 0.0 && policy.min_factor < 1.0)) - refuse("min_factor", policy.min_factor); + refuse_policy("min_factor", policy.min_factor); if (!(policy.rejection_growth > 0.0 && policy.rejection_growth <= 1.0)) - refuse("rejection_growth", policy.rejection_growth); + refuse_policy("rejection_growth", policy.rejection_growth); if (!(policy.hysteresis >= 1.0)) - refuse("hysteresis", policy.hysteresis); + refuse_policy("hysteresis", policy.hysteresis); if (policy.restart_after < 1) - refuse("restart_after", static_cast(policy.restart_after)); + refuse_policy("restart_after", static_cast(policy.restart_after)); if (!(policy.collar_stability > 0.0)) - refuse("collar_stability", policy.collar_stability); + refuse_policy("collar_stability", policy.collar_stability); if (!(policy.collar_rate >= 0.0)) - refuse("collar_rate", policy.collar_rate); + refuse_policy("collar_rate", policy.collar_rate); if (!(policy.carrier_cap > 0.0)) - refuse("carrier_cap", policy.carrier_cap); + refuse_policy("carrier_cap", policy.carrier_cap); if (policy.has_budget && policy.max_calls < 1) - refuse("max_calls", static_cast(policy.max_calls)); + refuse_policy("max_calls", static_cast(policy.max_calls)); if (!(policy.dz_min >= 0.0 && policy.dz_min <= policy.dz_max) || !std::isfinite(policy.dz_max)) - refuse("dz_min", policy.dz_min); + refuse_policy("dz_min", policy.dz_min); if (!(policy.stretch >= 1.0) || !std::isfinite(policy.stretch)) - refuse("stretch", policy.stretch); + refuse_policy("stretch", policy.stretch); if (!std::isfinite(policy.z_end)) - refuse("z_end", policy.z_end); + refuse_policy("z_end", policy.z_end); if (policy.breakpoints.size() < 2 || !increasing(policy.breakpoints)) - refuse("breakpoints", static_cast(policy.breakpoints.size())); + refuse_policy("breakpoints", static_cast(policy.breakpoints.size())); if (!increasing(policy.save_planes)) - refuse("save_planes", static_cast(policy.save_planes.size())); + refuse_policy("save_planes", static_cast(policy.save_planes.size())); if (!increasing(policy.hard_kinks)) - refuse("hard_kinks", static_cast(policy.hard_kinks.size())); + refuse_policy("hard_kinks", static_cast(policy.hard_kinks.size())); if (policy.landing_ulps < 1) - refuse("landing_ulps", policy.landing_ulps); + refuse_policy("landing_ulps", policy.landing_ulps); if (policy.has_quiet_interval && !(policy.quiet_interval > 0.0)) - refuse("quiet_interval", policy.quiet_interval); + refuse_policy("quiet_interval", policy.quiet_interval); } double factor(const Policy& policy, const double norm, const std::size_t order, @@ -186,7 +184,7 @@ OrderChoice choose_order(const Policy& policy, const Stencil& stencil, const double* norms, const double dz, const bool quiet) { const std::size_t current = stencil.order; if (!policy.variable_order) - return {policy.order, current, norms[0]}; + return {.stored = policy.order, .control = current, .norm = norms[0]}; std::array weights{}; for (std::size_t index = 0; index < stencil.n_orders; ++index) weights[index] = @@ -197,8 +195,10 @@ OrderChoice choose_order(const Policy& policy, const Stencil& stencil, if (weights[index] > weights[best]) best = index; if (stencil.orders[best] != current && weights[best] > policy.hysteresis * weights[0]) - return {stencil.orders[best], stencil.orders[best], norms[best]}; - return {current, current, norms[0]}; + return {.stored = stencil.orders[best], + .control = stencil.orders[best], + .norm = norms[best]}; + return {.stored = current, .control = current, .norm = norms[0]}; } Rejection rejection(const Policy& policy, const double norm, const std::size_t order, @@ -208,7 +208,7 @@ Rejection rejection(const Policy& policy, const double norm, const std::size_t o const double scale = !std::isfinite(norm) ? policy.min_factor : factor(policy, norm, order, policy.rejection_growth); - return {scale, restart ? 0 : count, restart}; + return {.factor = scale, .rejections = restart ? 0 : count, .restart = restart}; } double carrier_capped(const Policy& policy, const double z, const double proposal, @@ -258,17 +258,14 @@ double collar_reentry(const Policy& policy, const double proposal, Landing landing(const Policy& policy, const double z, const double dz, const double z_end) { if (!(dz > 0.0) || !std::isfinite(dz)) - throw std::invalid_argument("dz must be finite and positive, got " + - std::to_string(dz)); + refuse("dz must be finite and positive, got ", dz); const auto& edges = policy.breakpoints; if (!(edges.front() <= z && z <= edges.back())) - throw std::invalid_argument("z = " + std::to_string(z) + - " m lies outside the operator table"); + refuse("z = ", z, " m lies outside the operator table"); const double tolerance = policy.landing_ulps * ulp(py_max(std::fabs(z), std::fabs(z_end))); if (z >= z_end - tolerance) - throw std::invalid_argument("z = " + std::to_string(z) + - " m is already at or past z_end"); + refuse("z = ", z, " m is already at or past z_end"); const std::size_t found = bisect_right(edges, z); std::size_t index = found == 0 ? 0 : found - 1; index = std::min(index, edges.size() - 2); @@ -287,15 +284,16 @@ Landing landing(const Policy& policy, const double z, const double dz, return false; return static_cast(candidate.target) > static_cast(held.target); }; - Bound cap{dz, Target::none, 0.0}; - for (const Bound& other : {Bound{policy.dz_max, Target::max_step, 0.0}, - Bound{width, Target::interval_width, 0.0}}) + Bound cap{.dz = dz, .target = Target::none, .at = 0.0}; + for (const Bound& other : + {Bound{.dz = policy.dz_max, .target = Target::max_step, .at = 0.0}, + Bound{.dz = width, .target = Target::interval_width, .at = 0.0}}) if (better(other, cap)) cap = other; bool any = false; Bound land{}; const auto consider = [&](const Target target, const double position) { - const Bound candidate{position - z, target, position}; + const Bound candidate{.dz = position - z, .target = target, .at = position}; if (!any || better(candidate, land)) { land = candidate; any = true; @@ -310,10 +308,10 @@ Landing landing(const Policy& policy, const double z, const double dz, if (!(z_end < from)) consider(Target::end, z_end); if (land.dz <= cap.dz) - return {land.dz, land.target, true, land.at}; + return {.dz = land.dz, .target = land.target, .landed = true, .z_target = land.at}; if (land.dz <= py_min(policy.stretch * cap.dz, policy.dz_max)) - return {land.dz, land.target, true, land.at}; - return {cap.dz, cap.target, false, 0.0}; + return {.dz = land.dz, .target = land.target, .landed = true, .z_target = land.at}; + return {.dz = cap.dz, .target = cap.target, .landed = false, .z_target = 0.0}; } bool next_landing(const Policy& policy, const double z, const double target, @@ -340,7 +338,7 @@ Approach approach(const Policy& policy, const double z, const Landing& stop, if (next_landing(policy, z, target, ahead) && stop.dz < ahead - z && ahead - z < 2.0 * stop.dz) { const double half = 0.5 * (ahead - z); - return {half, z + half, false}; + return {.dz = half, .end = z + half, .kink = false}; } } const double end = stop.landed ? stop.z_target : z + stop.dz; @@ -353,7 +351,8 @@ Approach approach(const Policy& policy, const double z, const Landing& stop, on_kink = true; break; } - return {stop.dz, end, stop.target == Target::hard_kink || on_kink}; + return { + .dz = stop.dz, .end = end, .kink = stop.target == Target::hard_kink || on_kink}; } double proposal(const Policy& policy, const double dz, const double factor) { @@ -361,8 +360,9 @@ double proposal(const Policy& policy, const double dz, const double factor) { } bool replay_kink(const Policy& policy, const double end, const double slack) { - return std::any_of(policy.hard_kinks.begin(), policy.hard_kinks.end(), - [&](const double kink) { return std::fabs(kink - end) <= slack; }); + return std::ranges::any_of(policy.hard_kinks, [&](const double kink) { + return std::fabs(kink - end) <= slack; + }); } bool reached(const double position, const double target, const int ulps) { @@ -400,7 +400,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou const double z_end = policy.interpolate ? policy.z_end : target; std::uint64_t accepted = 0; std::array norms{}; - const auto elapsed = [&]() { + const auto elapsed = [&] { return std::chrono::duration(Clock::now() - started).count(); }; const auto record = [&](const double end, const double dz, const double norm, @@ -413,7 +413,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou // A replay runs while the schedule has steps and its target is not within the // slack (`HostAdamsMarch._replay_to`); an adaptive march until the target is reached. const std::size_t n_steps = schedule.steps.size(); - const auto running = [&]() { + const auto running = [&] { if (bounds.replay) return state.cursor < n_steps && state.z < target - bounds.slack; return !reached(state.z, target, policy.landing_ulps); @@ -428,8 +428,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou const auto take_check = [&]() -> watch::Verdict { if (bounds.owner != nullptr) return watch::owned_check( - *bounds.owner, [&engine]() { return engine.reading(); }, state.z, - bounds.replay, schedule.steps, state.cursor); + *bounds.owner, [&engine] { return engine.reading(); }, state.z, bounds.replay, + schedule.steps, state.cursor); const auto& rules = bounds.watch_policy; if (!watch::due(rules, state.watch, state.z)) { watch::skip(state.watch); @@ -474,7 +474,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou fail(result, Stage::startup, error); return result; } - state.history = {Entry{state.z, Identity::accepted, 0}}; + state.history = {Entry{.z = state.z, .identity = Identity::accepted, .row = 0}}; state.order = policy.variable_order ? 1 : policy.order; } Approach step; @@ -554,8 +554,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou state.history.end()); std::size_t free = 0; for (;; ++free) - if (std::none_of(kept.begin(), kept.end(), - [free](const Entry& entry) { return entry.row == free; })) + if (std::ranges::none_of( + kept, [free](const Entry& entry) { return entry.row == free; })) break; try { engine.accept(step.dz, free, trial, kept); @@ -563,7 +563,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou fail(result, Stage::accept, error); return result; } - kept.push_back(Entry{state.z + step.dz, Identity::predicted, free}); + kept.push_back( + Entry{.z = state.z + step.dz, .identity = Identity::predicted, .row = free}); state.history = std::move(kept); ++state.ledger.accepted_steps; state.rejections = 0; diff --git a/native/src/benchmark_gemm.cpp b/native/src/benchmark_gemm.cpp deleted file mode 100644 index c55501540..000000000 --- a/native/src/benchmark_gemm.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "solprop_native/gemm.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -using ExecutionSpace = Kokkos::DefaultExecutionSpace; -using Layout = Kokkos::LayoutRight; -using Range2 = Kokkos::MDRangePolicy< - ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Right, Kokkos::Iterate::Right>>; -using MemorySpace = ExecutionSpace::memory_space; -using Complex = Kokkos::complex; -using RealView = Kokkos::View; -using ComplexView = Kokkos::View; - -struct Shape { - std::size_t output_rows{}; - std::size_t inner{}; - std::size_t samples{}; -}; - -template KOKKOS_INLINE_FUNCTION Value initial_value(const double value) { - if constexpr (std::is_same_v) { - return Complex(value, -0.5 * value); - } else { - return value; - } -} - -template -void initialise(const ExecutionSpace& execution_space, const View& view, - const double seed) { - Kokkos::parallel_for( - "solprop_native.benchmark.initialise", - Range2(execution_space, {0, 0}, {view.extent(0), view.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - using Value = View::non_const_value_type; - const double value = seed + 0.01 * static_cast(row) + - 0.001 * static_cast(column); - view(row, column) = initial_value(value); - }); -} - -template -double measure(const ExecutionSpace& execution_space, const std::size_t repetitions, - Function&& function) { - for (std::size_t index = 0; index < 3; ++index) - function(); - execution_space.fence("solprop_native GEMM benchmark warmup"); - const auto started = std::chrono::steady_clock::now(); - for (std::size_t index = 0; index < repetitions; ++index) - function(); - execution_space.fence("solprop_native GEMM benchmark boundary"); - return std::chrono::duration(std::chrono::steady_clock::now() - - started) - .count() / - static_cast(repetitions); -} - -template double maximum_difference(const View& left, const View& right) { - double result = 0.0; - Kokkos::parallel_reduce( - "solprop_native.benchmark.compare", - Range2(ExecutionSpace{}, {0, 0}, {left.extent(0), left.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column, double& update) { - update = - Kokkos::max(update, Kokkos::abs(left(row, column) - right(row, column))); - }, - Kokkos::Max(result)); - return result; -} - -template -void team_gemm(const ExecutionSpace& execution_space, const AView& a, const BView& b, - const CView& c, const typename CView::non_const_value_type beta, - const int team_size) { - using Value = CView::non_const_value_type; - const std::size_t inner = a.extent(1); - const std::size_t samples = c.extent(1); - Kokkos::parallel_for( - "solprop_native.benchmark.team_gemm", - Kokkos::TeamPolicy(execution_space, static_cast(c.extent(0)), - team_size), - KOKKOS_LAMBDA( - const typename Kokkos::TeamPolicy::member_type& team) { - const std::size_t row = static_cast(team.league_rank()); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, samples), - [&](const std::size_t column) { - Value value{}; - for (std::size_t index = 0; index < inner; ++index) - value += a(row, index) * b(index, column); - c(row, column) = value + beta * c(row, column); - }); - }); -} - -double maximum_stacked_difference(const RealView& fine, const RealView& coarse, - const RealView& stacked) { - double result = 0.0; - const auto rows = fine.extent(0); - Kokkos::parallel_reduce( - "solprop_native.benchmark.compare_stacked", - Range2(ExecutionSpace{}, {0, 0}, {2 * rows, fine.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column, double& update) { - const double expected = - row < rows ? fine(row, column) : coarse(row - rows, column); - update = Kokkos::max(update, Kokkos::abs(expected - stacked(row, column))); - }, - Kokkos::Max(result)); - return result; -} - -std::size_t parsed(const char* value, const std::string_view name) { - const auto result = std::strtoull(value, nullptr, 10); - if (result == 0) { - std::cerr << name << " must be a positive integer\n"; - std::exit(2); - } - return static_cast(result); -} - -void run(const ExecutionSpace& execution_space, const std::string_view operation, - const Shape shape, const std::size_t repetitions, const double beta) { - RealView a("solprop_native.benchmark.a", shape.output_rows, shape.inner); - RealView b("solprop_native.benchmark.b", shape.inner, shape.samples); - RealView custom("solprop_native.benchmark.custom", shape.output_rows, shape.samples); - RealView blas("solprop_native.benchmark.blas", shape.output_rows, shape.samples); - initialise(execution_space, a, 0.25); - initialise(execution_space, b, -0.125); - Kokkos::deep_copy(execution_space, custom, 0.0); - Kokkos::deep_copy(execution_space, blas, 0.0); - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - solprop_native::kokkosblas_gemm(execution_space, a, b, blas, beta); - execution_space.fence("solprop_native GEMM benchmark reference"); - const double difference = maximum_difference(custom, blas); - const double custom_us = measure(execution_space, repetitions, [&] { - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - }); - const double blas_us = measure(execution_space, repetitions, [&] { - solprop_native::kokkosblas_gemm(execution_space, a, b, blas, beta); - }); - std::cout << operation << ",real," << shape.output_rows << ',' << shape.inner << ',' - << shape.samples << ',' << beta << ',' << repetitions << ',' - << std::setprecision(9) << custom_us << ',' << blas_us << ',' - << custom_us / blas_us << ',' << difference << '\n'; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const std::array team_sizes{64, 128, 256, 512}; -#else - const std::array team_sizes{1, 2, 4, 8}; -#endif - for (const int team_size : team_sizes) { - Kokkos::deep_copy(execution_space, custom, 0.0); - Kokkos::deep_copy(execution_space, blas, 0.0); - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - team_gemm(execution_space, a, b, blas, beta, team_size); - execution_space.fence("solprop_native team GEMM benchmark reference"); - const double team_difference = maximum_difference(custom, blas); - const double team_us = measure(execution_space, repetitions, [&] { - team_gemm(execution_space, a, b, blas, beta, team_size); - }); - std::cout << operation << "-team" << team_size << ",real," << shape.output_rows - << ',' << shape.inner << ',' << shape.samples << ',' << beta << ',' - << repetitions << ',' << custom_us << ',' << team_us << ',' - << custom_us / team_us << ',' << team_difference << '\n'; - } - - ComplexView complex_a("solprop_native.benchmark.complex_a", shape.output_rows, - shape.inner); - ComplexView complex_b("solprop_native.benchmark.complex_b", shape.inner, - shape.samples); - ComplexView mixed("solprop_native.benchmark.mixed", shape.output_rows, shape.samples); - ComplexView complex_blas("solprop_native.benchmark.complex_blas", shape.output_rows, - shape.samples); - initialise(execution_space, complex_b, -0.125); - Kokkos::parallel_for( - "solprop_native.benchmark.promote_real_matrix", - Range2(execution_space, {0, 0}, {a.extent(0), a.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - complex_a(row, column) = Complex(a(row, column), 0.0); - }); - Kokkos::deep_copy(execution_space, mixed, Complex{}); - Kokkos::deep_copy(execution_space, complex_blas, Complex{}); - solprop_native::custom_gemm(execution_space, a, complex_b, mixed, Complex(beta)); - solprop_native::kokkosblas_gemm(execution_space, complex_a, complex_b, complex_blas, - Complex(beta)); - execution_space.fence("solprop_native complex GEMM benchmark reference"); - const double complex_difference = maximum_difference(mixed, complex_blas); - const double mixed_us = measure(execution_space, repetitions, [&] { - solprop_native::custom_gemm(execution_space, a, complex_b, mixed, Complex(beta)); - }); - const double complex_blas_us = measure(execution_space, repetitions, [&] { - solprop_native::kokkosblas_gemm(execution_space, complex_a, complex_b, complex_blas, - Complex(beta)); - }); - std::cout << operation << ",mixed-vs-complex," << shape.output_rows << ',' - << shape.inner << ',' << shape.samples << ',' << beta << ',' << repetitions - << ',' << mixed_us << ',' << complex_blas_us << ',' - << mixed_us / complex_blas_us << ',' << complex_difference << '\n'; -} - -void run_transform_pair(const ExecutionSpace& execution_space, const std::size_t nodes, - const std::size_t modes, const std::size_t samples, - const std::size_t repetitions) { - run(execution_space, "synthesis-overwrite", {nodes, modes, samples}, repetitions, - 0.0); - run(execution_space, "projection-overwrite", {modes, nodes, samples}, repetitions, - 0.0); - run(execution_space, "projection-accumulate", {modes, nodes, samples}, repetitions, - 1.0); - run(execution_space, "fine-coarse-stacked", {2 * modes, nodes, samples}, repetitions, - 0.0); - - RealView fine_matrix("solprop_native.benchmark.fine_matrix", modes, nodes); - RealView coarse_matrix("solprop_native.benchmark.coarse_matrix", modes, nodes); - RealView stacked_matrix("solprop_native.benchmark.stacked_matrix", 2 * modes, nodes); - RealView response("solprop_native.benchmark.response", nodes, samples); - RealView fine("solprop_native.benchmark.fine", modes, samples); - RealView coarse("solprop_native.benchmark.coarse", modes, samples); - RealView stacked("solprop_native.benchmark.stacked", 2 * modes, samples); - initialise(execution_space, fine_matrix, 0.25); - initialise(execution_space, coarse_matrix, -0.5); - initialise(execution_space, response, -0.125); - Kokkos::parallel_for( - "solprop_native.benchmark.stack_projection", - Range2(execution_space, {0, 0}, {2 * modes, nodes}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - stacked_matrix(row, column) = - row < modes ? fine_matrix(row, column) : coarse_matrix(row - modes, column); - }); - const auto separate = [&] { - solprop_native::kokkosblas_gemm(execution_space, fine_matrix, response, fine, 0.0); - solprop_native::kokkosblas_gemm(execution_space, coarse_matrix, response, coarse, - 0.0); - }; - const auto combined = [&] { - solprop_native::kokkosblas_gemm(execution_space, stacked_matrix, response, stacked, - 0.0); - }; - separate(); - combined(); - execution_space.fence("solprop_native stacked projection reference"); - const double difference = maximum_stacked_difference(fine, coarse, stacked); - const double separate_us = measure(execution_space, repetitions, separate); - const double stacked_us = measure(execution_space, repetitions, combined); - std::cout << "fine-coarse-dispatch,real," << 2 * modes << ',' << nodes << ',' - << samples << ",0," << repetitions << ',' << separate_us << ',' - << stacked_us << ',' << separate_us / stacked_us << ',' << difference - << '\n'; -} - -} // namespace - -int main(int argc, char** argv) { - Kokkos::ScopeGuard guard(argc, argv); - const ExecutionSpace execution_space; - std::cout << "operation,kind,rows,inner,samples,beta,repetitions,baseline_us," - "candidate_us,speedup,max_abs_diff\n"; - if (argc == 5) { - run_transform_pair(execution_space, parsed(argv[1], "nodes"), - parsed(argv[2], "modes"), parsed(argv[3], "samples"), - parsed(argv[4], "repetitions")); - return 0; - } - const std::vector modes{1, 2, 4, 6, 8}; - const std::vector rows{1, 8, 21, 33, 41, 64, 96}; - const std::vector samples{1024, 4096, 8192, 32768, 131072}; - for (const auto mode : modes) { - for (const auto row : rows) { - for (const auto count : samples) { - const std::size_t repetitions = - std::max(7, 1048576 / std::max(count, 1)); - run_transform_pair(execution_space, row, mode, count, repetitions); - } - } - } -} diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index b6a13c8fe..28b1d1b10 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -12,6 +12,7 @@ #include "solprop_native/fft_plan.hpp" #include "solprop_native/lagrange_basis.hpp" #include "solprop_native/low_storage.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" @@ -20,13 +21,14 @@ #include +#include #include // `available_device_memory` calls the vendor runtime directly. Kokkos pulls // these headers in transitively today; declaring them here is what makes the // probe's own dependency explicit rather than a property of a Kokkos version. // The guards resolve only after Kokkos_Core.hpp has defined them. -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA #include #elif defined(KOKKOS_ENABLE_HIP) #include @@ -40,13 +42,12 @@ #include #include -#if defined(_WIN32) +#ifdef _WIN32 #include #else #include #endif -#include #include #include #include @@ -197,9 +198,8 @@ native::CpuExecution cpu_execution_from(const nb::dict& values) { const std::size_t expected = 2 * native::cpu_operation_count + cpu_execution_scalars.size(); if (values.size() != expected) - throw std::invalid_argument("a CPU execution record has " + - std::to_string(expected) + " fields, got " + - std::to_string(values.size())); + native::refuse("a CPU execution record has ", expected, " fields, got ", + values.size()); const auto field = [&values](const std::string& name) { if (!values.contains(name.c_str())) throw std::invalid_argument("a CPU execution record misses a field: " + name); @@ -208,8 +208,9 @@ native::CpuExecution cpu_execution_from(const nb::dict& values) { try { for (std::size_t index = 0; index < native::cpu_operation_count; ++index) { const std::string name = cpu_operation_names[index]; - result.grain[index] = {nb::cast(field(name + "_threshold")), - nb::cast(field(name + "_full"))}; + result.grain[index] = {.threshold = + nb::cast(field(name + "_threshold")), + .full = nb::cast(field(name + "_full"))}; } result.response_full_samples = nb::cast(field("response_full_samples")); @@ -315,6 +316,24 @@ nb::dict fft_planning_dict(const native::FftPlanning& planning) { return result; } +// The planning a workspace is asked for, as `fft_planning_from` reads it back. +nb::dict fft_request_dict(const native::FftPlanning& planning) { + static constexpr std::array modes{"legacy", "profile", "capture"}; + nb::dict efforts; + efforts["rows"] = planning.rows; + efforts["threaded_rows"] = planning.threaded_rows; + efforts["planes"] = planning.planes; + nb::dict result; + result["mode"] = modes[static_cast(planning.mode)]; + result["efforts"] = efforts; + result["wisdom"] = planning.wisdom; + result["capture_effort"] = planning.capture_effort; + result["time_limit"] = planning.time_limit; + result["stored"] = planning.stored; + result["keep"] = planning.keep; + return result; +} + nb::dict cpu_execution_dict(const native::CpuExecution& execution) { nb::dict result; for (std::size_t index = 0; index < native::cpu_operation_count; ++index) { @@ -333,411 +352,370 @@ nb::dict cpu_execution_dict(const native::CpuExecution& execution) { return result; } -native::Problem problem_from(const nb::dict& values) { - native::Problem result; - if (!values.contains("cpu_execution")) - throw std::invalid_argument("a native problem needs its CPU execution record " - "(cpu_execution, solprop SEP 46)"); - try { - result.cpu_execution = - cpu_execution_from(nb::cast(values["cpu_execution"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("a native problem's cpu_execution must be a dict"); - } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if (values.contains("fft_planning")) { - try { - result.fft_planning = - fft_planning_from(nb::cast(values["fft_planning"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("a native problem's fft_planning must be a dict"); - } - } -#endif - try { - if (values.contains("n_x")) { - result.n_x = nb::cast(values["n_x"]); - result.n_y = nb::cast(values["n_y"]); - if (values.contains("n_p")) - result.n_p = nb::cast(values["n_p"]); - if (values.contains("spatial_far_field_slots")) - result.spatial_far_field_slots = - nb::cast(values["spatial_far_field_slots"]); - } - result.n_omega = nb::cast(values["n_omega"]); - result.n_t = nb::cast(values["n_t"]); - result.n_to = nb::cast(values["n_to"]); - result.n_stages = nb::cast(values["n_stages"]); - if (values.contains("n_modes")) { - result.n_modes = nb::cast(values["n_modes"]); - result.n_pol_state = nb::cast(values["n_pol_state"]); - result.n_pol_transform = nb::cast(values["n_pol_transform"]); - result.n_nodes = nb::cast(values["n_nodes"]); - result.node_chunk = nb::cast(values["node_chunk"]); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem dimensions must be integers"); - } - try { - result.family = field_family(nb::cast(values["family"])); - result.nonlinear = nb::cast(values["nonlinear"]); - result.modified_noise = nb::cast(values["modified_noise"]); - result.in_stage_absorber = nb::cast(values["in_stage_absorber"]); - result.split_step_absorber = nb::cast(values["split_step_absorber"]); - result.first_same_as_last = nb::cast(values["first_same_as_last"]); - if (values.contains("low_storage")) { - result.low_storage = nb::cast(values["low_storage"]); - result.low_storage_stage_register = - nb::cast(values["low_storage_stage_register"]); - result.low_storage_flows = nb::cast(values["low_storage_flows"]); - result.low_storage_tiled_input = - nb::cast(values["low_storage_tiled_input"]); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem family and flags have wrong types"); +// ------------------------------------------------------------- the problem's fields +// A vector field of a bound struct crosses as an exact NumPy array: float64, +// complex128, uint64 or bool, C-contiguous on the CPU and of any rank, copied in, and +// out as a one-dimensional copy. Anything else is refused, naming the field. + +template struct ExactArray; +template <> struct ExactArray { + using Input = RealInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = double; + static constexpr const char* dtype = "float64"; + static constexpr const char* scalar = "numpy.float64"; +}; +template <> struct ExactArray { + using Input = ComplexInput; + using Output = + nb::ndarray, nb::ndim<1>, nb::c_contig>; + using Stored = std::complex; + static constexpr const char* dtype = "complex128"; + static constexpr const char* scalar = "numpy.complex128"; +}; +template <> struct ExactArray { + using Input = IndexInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = std::uint64_t; + static constexpr const char* dtype = "uint64"; + static constexpr const char* scalar = "numpy.uint64"; +}; +template <> struct ExactArray { + using Input = BoolInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = bool; + static constexpr const char* dtype = "bool"; + static constexpr const char* scalar = "numpy.bool_"; +}; + +template +std::vector exact_vector(const nb::handle value, const std::string& field, + const std::size_t rank) { + typename ExactArray::Input array; + if (!nb::try_cast(value, array, false) || (rank != 0 && array.ndim() != rank)) + native::refuse(field, " must be a C-contiguous ", ExactArray::dtype, + " array", rank == 0 ? "" : native::concat(" of rank ", rank)); + if constexpr (std::is_same_v) + return real_vector(array); + else if constexpr (std::is_same_v) + return complex_vector(array); + else if constexpr (std::is_same_v) + return index_vector(array); + else + return bool_vector(array); +} + +template +ExactArray::Output array_of(const std::vector& values) { + using Stored = ExactArray::Stored; + auto storage = std::make_unique(values.size()); + for (std::size_t index = 0; index < values.size(); ++index) { + if constexpr (std::is_same_v) + storage[index] = {values[index].real(), values[index].imag()}; + else + storage[index] = static_cast(values[index]); } - try { - result.crop_index = index_vector(exact_array(values["crop_index"])); - result.active_mask = bool_vector(exact_array(values["active_mask"])); - result.norm_mask = bool_vector(exact_array(values["norm_mask"])); - result.omega_window = real_vector(exact_array(values["omega_window"])); - if (values.contains("omega")) - result.omega = real_vector(exact_array(values["omega"])); - if (values.contains("power_weight")) - result.power_weight = real_vector(exact_array(values["power_weight"])); - result.time_window = real_vector(exact_array(values["time_window"])); - result.time_window_derivative = - real_vector(exact_array(values["time_window_derivative"])); - result.absorber_fine = real_vector(exact_array(values["absorber_fine"])); - result.absorber_coarse = - real_vector(exact_array(values["absorber_coarse"])); - if (result.separable_geometry()) { - result.spatial_evanescent_rate = - nb::cast(values["spatial_evanescent_rate"]); - result.spatial_k_squared = - complex_vector(exact_array(values["spatial_k_squared"])); - result.spatial_k_squared_low = - complex_vector(exact_array(values["spatial_k_squared_low"])); - result.spatial_numerator = - complex_vector(exact_array(values["spatial_numerator"])); - result.spatial_clamped = - complex_vector(exact_array(values["spatial_clamped"])); - result.spatial_kperp_sq = - real_vector(exact_array(values["spatial_kperp_sq"])); - result.spatial_kperp_sq_low = - real_vector(exact_array(values["spatial_kperp_sq_low"])); - result.spatial_frame = - real_vector(exact_array(values["spatial_frame"])); - result.spatial_spectral_rate = - real_vector(exact_array(values["spatial_spectral_rate"])); - result.spatial_user_rate = - real_vector(exact_array(values["spatial_user_rate"])); - result.spatial_absorber_rate = - real_vector(exact_array(values["spatial_absorber_rate"])); - result.spatial_kperp_rate = - real_vector(exact_array(values["spatial_kperp_rate"])); - if (result.slab_geometry()) - result.spatial_kappa_sq = - complex_vector(exact_array(values["spatial_kappa_sq"])); - if (result.free_geometry()) { - result.spatial_k_x = real_vector(exact_array(values["spatial_k_x"])); - result.spatial_k_y = real_vector(exact_array(values["spatial_k_y"])); - } - } - if (values.contains("spatial_absorber_matrix")) - result.spatial_absorber_matrix = - real_vector(exact_array(values["spatial_absorber_matrix"])); - if (values.contains("spatial_spectral_weights")) { - result.spatial_spectral_weights = - real_vector(exact_array(values["spatial_spectral_weights"])); - result.spatial_on_axis = - complex_vector(exact_array(values["spatial_on_axis"])); - } - if (values.contains("flux_k_squared")) { - result.flux_k_squared = - complex_vector(exact_array(values["flux_k_squared"])); - result.flux_k_squared_low = - complex_vector(exact_array(values["flux_k_squared_low"])); - result.flux_kperp_sq = - real_vector(exact_array(values["flux_kperp_sq"])); - result.flux_kperp_sq_low = - real_vector(exact_array(values["flux_kperp_sq_low"])); - result.flux_table = nb::cast(values["flux_table"]); - } - if (values.contains("flux_closure")) - result.flux_closure = nb::cast(values["flux_closure"]); - result.carrier = complex_vector(exact_array(values["carrier"])); - if (values.contains("synthesis_matrix")) { - result.synthesis_matrix = - real_vector(exact_array(values["synthesis_matrix"])); - result.projection_matrix = - real_vector(exact_array(values["projection_matrix"])); - result.projection_coarse_matrix = - real_vector(exact_array(values["projection_coarse_matrix"])); + nb::capsule owner(storage.get(), [](void* held) noexcept { + const std::unique_ptr owned(static_cast(held)); + }); + const std::array shape{values.size()}; + return {storage.release(), 1, shape.data(), owner}; +} + +template struct IsVector : std::false_type {}; +template struct IsVector> : std::true_type {}; + +/** One group of the problem's fields, bound as a nested class: scalars read and + written as they are, vectors as exact arrays (`exact_vector`). */ +template class Fields { +public: + Fields(nb::handle scope, const char* name, std::string path) + : class_(scope, name), path_(std::move(path)) { + class_.def(nb::init<>()); + } + /** Bind `member` as `name`; a vector field of a nonzero `rank` takes arrays of that + rank alone, as the extents a count beside it gives are laid out. */ + template + Fields& field(const char* name, Value Group::* member, const std::size_t rank = 0) { + if constexpr (IsVector::value) { + using Element = Value::value_type; + // The setter's stub names the one dtype it takes. + const std::string setter = + native::concat("def ", name, "(self, arg: numpy.typing.NDArray[", + ExactArray::scalar, "], /) -> None"); + class_.def_prop_rw( + name, [member](const Group& self) { return array_of(self.*member); }, + [member, rank, path = path_ + "." + name](Group& self, + const nb::handle value) { + self.*member = exact_vector(value, path, rank); + }, + nb::for_getter(nb::rv_policy::automatic), + nb::for_setter(nb::sig(setter.c_str()))); + } else { + static_cast(rank); + class_.def_rw(name, member); } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem masks and transform arrays have wrong dtypes"); - } - ComplexInput phi; - ComplexInput prefactor; - try { - phi = exact_array(values["phi_terms"]); - prefactor = exact_array(values["prefactor_terms"]); - } catch (const nb::cast_error&) { - throw std::invalid_argument("fixed terms must be complex128 arrays"); - } - if ((phi.ndim() != 2 && phi.ndim() != 3) || - (prefactor.ndim() != 2 && prefactor.ndim() != 3)) { - throw std::invalid_argument("fixed terms must be rank-two or rank-three arrays"); - } - result.n_phi_terms = phi.shape(0); - result.n_prefactor_terms = prefactor.shape(0); - result.phi_terms = complex_vector(phi); - result.prefactor_terms = complex_vector(prefactor); - // solprop SEP 29 G1: a z table's terms are slots its evaluation fills; the leading - // extent of an empty terms array names how many. - try { - if (values.contains("phi_table")) - result.phi_table = nb::cast(values["phi_table"]); - if (values.contains("prefactor_table")) - result.prefactor_table = nb::cast(values["prefactor_table"]); - } catch (const nb::cast_error&) { - throw std::invalid_argument("z table flags must be booleans"); - } - try { - result.tableau_matrix = - real_vector(exact_array(values["tableau_matrix"])); - result.tableau_nodes = real_vector(exact_array(values["tableau_nodes"])); - result.solution_weights = - real_vector(exact_array(values["solution_weights"])); - result.error_weights = real_vector(exact_array(values["error_weights"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("tableau values must be float64 arrays"); + return *this; } - try { - result.synthesis = nb::cast(values["synthesis"]); - result.projection = nb::cast(values["projection"]); - result.kerr_coefficient = nb::cast(values["kerr_coefficient"]); - result.self_factor = nb::cast(values["self_factor"]); - result.conjugate_factor = nb::cast(values["conjugate_factor"]); - result.harmonic_factor = nb::cast(values["harmonic_factor"]); - result.take_real = nb::cast(values["take_real"]); - if (values.contains("plasma")) { - result.plasma = nb::cast(values["plasma"]); - result.raman = nb::cast(values["raman"]); - result.raman_thg = nb::cast(values["raman_thg"]); - result.stash_plasma = nb::cast(values["stash_plasma"]); - result.weighted_controller = nb::cast(values["weighted_controller"]); - result.dt_fine = nb::cast(values["dt_fine"]); - result.dt = nb::cast(values["dt"]); - result.raman_fraction = nb::cast(values["raman_fraction"]); - result.raman_epsilon0 = nb::cast(values["raman_epsilon0"]); - const auto raman_undamped = exact_array(values["raman_undamped"]); - if (result.raman && (raman_undamped.ndim() != 2 || raman_undamped.shape(0) == 0 || - result.n_to > std::numeric_limits::max() / 2 || - raman_undamped.shape(1) != 2 * result.n_to)) - throw std::invalid_argument( - "raman_undamped must have shape (n_terms, 2 * n_to)"); - result.n_raman_terms = result.raman ? raman_undamped.shape(0) : 0; - if (result.raman) { - result.raman_undamped = real_vector(raman_undamped); - result.raman_tensor_capable = nb::cast(values["raman_tensor_capable"]); - const auto weights = exact_array(values["raman_tensor_weights"]); - if (weights.ndim() != 2 || weights.shape(0) != result.n_raman_terms || - weights.shape(1) != 2) - throw std::invalid_argument( - "raman_tensor_weights must have shape (n_terms, 2)"); - result.raman_tensor_weights = real_vector(weights); - } - result.charge_mass_ratio = nb::cast(values["charge_mass_ratio"]); - result.ionisation_potential = nb::cast(values["ionisation_potential"]); - result.log_field_min = nb::cast(values["log_field_min"]); - result.d_log_field = nb::cast(values["d_log_field"]); - result.field_min = nb::cast(values["field_min"]); - result.field_max = nb::cast(values["field_max"]); - result.rate_max = nb::cast(values["rate_max"]); - const auto coefficients = exact_array(values["rate_coefficients"]); - if (coefficients.ndim() != 2 || coefficients.shape(0) != 4) - throw std::invalid_argument( - "rate_coefficients must have shape (4, n_intervals)"); - result.rate_intervals = coefficients.shape(1); - result.rate_coefficients = real_vector(coefficients); - // solprop SEP 18: the running-integral rule. Translation designed and - // certified the taps; the core checks their extent, finiteness and the - // trapezoid's one identity tap. - if (values.contains("plasma_gated")) - result.plasma_gated = nb::cast(values["plasma_gated"]); - if (values.contains("plasma_band")) { - result.plasma_band = nb::cast(values["plasma_band"]); - const auto taps = exact_array(values["plasma_taps"]); - if (taps.ndim() != 1) - throw std::invalid_argument("plasma_taps must be one-dimensional"); - result.plasma_taps = real_vector(taps); - } - if (values.contains("n_channels")) { - result.n_channels = nb::cast(values["n_channels"]); - result.channel_parameters = - real_vector(exact_array(values["channel_parameters"])); - result.channel_coefficients = - real_vector(exact_array(values["channel_coefficients"])); - result.n_raman_components = nb::cast(values["n_raman_components"]); - result.raman_row_component = - real_vector(exact_array(values["raman_row_component"])); - result.raman_component_fraction = - real_vector(exact_array(values["raman_component_fraction"])); - } - if (values.contains("envelope_plasma")) { - result.envelope_plasma = nb::cast(values["envelope_plasma"]); - if (result.envelope_plasma) { - const auto nodes = exact_array(values["envelope_rule_nodes"]); - const auto weights = exact_array(values["envelope_rule_weights"]); - for (const auto& rule : {nodes, weights}) - if (rule.ndim() != 3 || rule.shape(0) != result.n_channels || - rule.shape(1) != 4 || rule.shape(2) != 512) + +private: + nb::class_ class_; + std::string path_; +}; + +void bind_problem(nb::module_& module_) { + using P = native::Problem; + nb::enum_(module_, "FieldFamily") + .value("field_real", native::FieldFamily::field_real) + .value("field_analytic", native::FieldFamily::field_analytic) + .value("envelope", native::FieldFamily::envelope); + nb::class_

problem(module_, "Problem"); + Fields(problem, "Dims", "dims") + .field("n_omega", &P::Dims::n_omega) + .field("n_t", &P::Dims::n_t) + .field("n_to", &P::Dims::n_to) + .field("n_modes", &P::Dims::n_modes) + .field("n_pol_state", &P::Dims::n_pol_state) + .field("n_pol_transform", &P::Dims::n_pol_transform) + .field("n_nodes", &P::Dims::n_nodes) + .field("node_chunk", &P::Dims::node_chunk) + .field("family", &P::Dims::family); + Fields(problem, "Grid", "grid") + .field("crop_index", &P::Grid::crop_index) + .field("active_mask", &P::Grid::active_mask) + .field("norm_mask", &P::Grid::norm_mask) + .field("omega_window", &P::Grid::omega_window) + .field("omega", &P::Grid::omega) + .field("power_weight", &P::Grid::power_weight) + .field("time_window", &P::Grid::time_window) + .field("time_window_derivative", &P::Grid::time_window_derivative) + .field("carrier", &P::Grid::carrier) + .field("dt_fine", &P::Grid::dt_fine) + .field("dt", &P::Grid::dt); + Fields(problem, "Spatial", "spatial") + .field("n_x", &P::Spatial::n_x) + .field("n_y", &P::Spatial::n_y) + .field("k_squared", &P::Spatial::k_squared) + .field("k_squared_low", &P::Spatial::k_squared_low) + .field("numerator", &P::Spatial::numerator) + .field("clamped", &P::Spatial::clamped) + .field("kperp_sq", &P::Spatial::kperp_sq) + .field("kperp_sq_low", &P::Spatial::kperp_sq_low) + .field("frame", &P::Spatial::frame) + .field("spectral_rate", &P::Spatial::spectral_rate) + .field("user_rate", &P::Spatial::user_rate) + .field("absorber_rate", &P::Spatial::absorber_rate) + .field("kperp_rate", &P::Spatial::kperp_rate) + .field("evanescent_rate", &P::Spatial::evanescent_rate) + .field("k_x", &P::Spatial::k_x) + .field("k_y", &P::Spatial::k_y) + .field("far_field_slots", &P::Spatial::far_field_slots) + .field("n_p", &P::Spatial::n_p) + .field("kappa_sq", &P::Spatial::kappa_sq) + .field("absorber_matrix", &P::Spatial::absorber_matrix) + .field("spectral_weights", &P::Spatial::spectral_weights) + .field("on_axis", &P::Spatial::on_axis); + Fields(problem, "Flux", "flux") + .field("k_squared", &P::Flux::k_squared) + .field("k_squared_low", &P::Flux::k_squared_low) + .field("kperp_sq", &P::Flux::kperp_sq) + .field("kperp_sq_low", &P::Flux::kperp_sq_low) + .field("table", &P::Flux::table) + .field("closure", &P::Flux::closure); + Fields(problem, "Stepper", "stepper") + .field("n_stages", &P::Stepper::n_stages) + .field("tableau_matrix", &P::Stepper::tableau_matrix) + .field("tableau_nodes", &P::Stepper::tableau_nodes) + .field("solution_weights", &P::Stepper::solution_weights) + .field("error_weights", &P::Stepper::error_weights) + .field("first_same_as_last", &P::Stepper::first_same_as_last) + .field("weighted_controller", &P::Stepper::weighted_controller) + .field("modified_noise", &P::Stepper::modified_noise) + .field("low_storage", &P::Stepper::low_storage) + .field("low_storage_stage_register", &P::Stepper::low_storage_stage_register) + .field("low_storage_flows", &P::Stepper::low_storage_flows) + .field("low_storage_tiled_input", &P::Stepper::low_storage_tiled_input); + Fields(problem, "Linear", "linear") + .field("n_phi_terms", &P::Linear::n_phi_terms) + .field("n_prefactor_terms", &P::Linear::n_prefactor_terms) + .field("phi_table", &P::Linear::phi_table) + .field("prefactor_table", &P::Linear::prefactor_table) + .field("phi_terms", &P::Linear::phi_terms) + .field("prefactor_terms", &P::Linear::prefactor_terms); + Fields(problem, "Absorber", "absorber") + .field("in_stage", &P::Absorber::in_stage) + .field("split_step", &P::Absorber::split_step) + .field("fine", &P::Absorber::fine) + .field("coarse", &P::Absorber::coarse); + Fields(problem, "Transform", "transform") + .field("synthesis_matrix", &P::Transform::synthesis_matrix) + .field("projection_matrix", &P::Transform::projection_matrix) + .field("projection_coarse_matrix", &P::Transform::projection_coarse_matrix) + .field("synthesis", &P::Transform::synthesis) + .field("projection", &P::Transform::projection); + Fields(problem, "Kerr", "kerr") + .field("nonlinear", &P::Kerr::nonlinear) + .field("coefficient", &P::Kerr::coefficient) + .field("self_factor", &P::Kerr::self_factor) + .field("conjugate_factor", &P::Kerr::conjugate_factor) + .field("harmonic_factor", &P::Kerr::harmonic_factor) + .field("take_real", &P::Kerr::take_real); + Fields(problem, "Plasma", "plasma") + .field("enabled", &P::Plasma::enabled) + .field("envelope", &P::Plasma::envelope) + .field("gated", &P::Plasma::gated) + .field("stash", &P::Plasma::stash) + .field("envelope_rule_nodes", &P::Plasma::envelope_rule_nodes, 3) + .field("envelope_rule_weights", &P::Plasma::envelope_rule_weights, 3) + .field("envelope_drude_factor", &P::Plasma::envelope_drude_factor, 1) + .field("envelope_loss_factor", &P::Plasma::envelope_loss_factor, 1) + .field("charge_mass_ratio", &P::Plasma::charge_mass_ratio) + .field("ionisation_potential", &P::Plasma::ionisation_potential) + .field("log_field_min", &P::Plasma::log_field_min) + .field("d_log_field", &P::Plasma::d_log_field) + .field("field_min", &P::Plasma::field_min) + .field("field_max", &P::Plasma::field_max) + .field("rate_max", &P::Plasma::rate_max) + .field("rate_intervals", &P::Plasma::rate_intervals) + .field("rate_coefficients", &P::Plasma::rate_coefficients, 2) + .field("n_channels", &P::Plasma::n_channels) + .field("band", &P::Plasma::band) + .field("taps", &P::Plasma::taps, 1) + .field("channel_parameters", &P::Plasma::channel_parameters) + .field("channel_coefficients", &P::Plasma::channel_coefficients); + Fields(problem, "Raman", "raman") + .field("enabled", &P::Raman::enabled) + .field("thg", &P::Raman::thg) + .field("tensor_capable", &P::Raman::tensor_capable) + .field("n_terms", &P::Raman::n_terms) + .field("fraction", &P::Raman::fraction) + .field("epsilon0", &P::Raman::epsilon0) + .field("undamped", &P::Raman::undamped, 2) + .field("tensor_weights", &P::Raman::tensor_weights, 2) + .field("n_components", &P::Raman::n_components) + .field("row_component", &P::Raman::row_component) + .field("component_fraction", &P::Raman::component_fraction); + Fields(problem, "General", "general") + .field("enabled", &P::General::enabled) + .field("real_form", &P::General::real_form) + .field("map_intervals", &P::General::map_intervals) + .field("map_n_deg", &P::General::map_n_deg) + .field("components", &P::General::components) + .field("nodes", &P::General::nodes) + .field("node_chunk", &P::General::node_chunk) + .field("responses", &P::General::responses) + .field("terms", &P::General::terms) + .field("pad_scale", &P::General::pad_scale) + .field("epsilon0", &P::General::epsilon0) + .field("synthesis", &P::General::synthesis) + .field("projection", &P::General::projection) + .field("frequency_factor", &P::General::frequency_factor) + .field("delayed_spectrum", &P::General::delayed_spectrum) + .field("tensor", &P::General::tensor) + .field("delayed_tensor", &P::General::delayed_tensor) + .field("node_response", &P::General::node_response); + Fields(problem, "Flow", "flow") + .field("enabled", &P::Flow::enabled) + .field("active", &P::Flow::active) + .field("sections", &P::Flow::sections) + .field("terms", &P::Flow::terms) + .field("slot", &P::Flow::slot) + .field("frozen", &P::Flow::frozen) + .field("residual_terms", &P::Flow::residual_terms) + .field("left", &P::Flow::left) + .field("right", &P::Flow::right) + .field("eigenvalues", &P::Flow::eigenvalues) + .field("route", &P::Flow::route) + .field("fallback_vectors", &P::Flow::fallback_vectors) + .field("fallback_triangular", &P::Flow::fallback_triangular); + Fields(problem, "Dispersive", "dispersive") + .field("enabled", &P::Dispersive::enabled) + .field("symmetric", &P::Dispersive::symmetric) + .field("form", &P::Dispersive::form) + .field("inputs", &P::Dispersive::inputs) + .field("term_multipliers", &P::Dispersive::term_multipliers) + .field("coefficients", &P::Dispersive::coefficients) + .field("pair_constants", &P::Dispersive::pair_constants) + .field("quartics", &P::Dispersive::quartics) + .field("input_scale", &P::Dispersive::input_scale) + .field("output_scale", &P::Dispersive::output_scale) + .field("term_rows", &P::Dispersive::term_rows) + .field("term_channels", &P::Dispersive::term_channels) + .field("group_offsets", &P::Dispersive::group_offsets) + .field("group_members", &P::Dispersive::group_members) + .field("group_keys", &P::Dispersive::group_keys) + .field("node_region", &P::Dispersive::node_region); + problem.def(nb::init<>()) + .def_rw("dims", &P::dims) + .def_rw("grid", &P::grid) + .def_rw("spatial", &P::spatial) + .def_rw("flux", &P::flux) + .def_rw("stepper", &P::stepper) + .def_rw("linear", &P::linear) + .def_rw("absorber", &P::absorber) + .def_rw("transform", &P::transform) + .def_rw("kerr", &P::kerr) + .def_rw("plasma", &P::plasma) + .def_rw("raman", &P::raman) + .def_rw("general", &P::general) + .def_rw("flow", &P::flow) + .def_rw("dispersive", &P::dispersive) + // solprop SEP 46: the CPU execution record and the FFT planning cross as the + // records solprop_native's `_execution` and `_wisdom` write. + .def_prop_rw( + "cpu_execution", + [](const P& self) { return cpu_execution_dict(self.cpu_execution); }, + [](P& self, const nb::handle values) { + if (!nb::isinstance(values)) throw std::invalid_argument( - "envelope rules require (n_channels, 4, 512)"); - const auto drude = exact_array(values["envelope_drude_factor"]); - const auto loss = exact_array(values["envelope_loss_factor"]); - if (drude.ndim() != 1 || drude.shape(0) != result.n_omega || - loss.ndim() != 1 || loss.shape(0) != result.n_omega) - throw std::invalid_argument("envelope current factors require (n_omega,)"); - result.envelope_rule_nodes = real_vector(nodes); - result.envelope_rule_weights = real_vector(weights); - result.envelope_drude_factor = real_vector(drude); - result.envelope_loss_factor = complex_vector(loss); - } - } - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("response scalars have wrong types"); - } - if (values.contains("dispersive")) { - try { - result.dispersive = nb::cast(values["dispersive"]); - if (result.dispersive) { - result.dispersive_symmetric = nb::cast(values["dispersive_symmetric"]); - result.dispersive_form = nb::cast(values["dispersive_form"]); - result.dispersive_inputs = - real_vector(exact_array(values["dispersive_inputs"])); - result.dispersive_term_multipliers = - real_vector(exact_array(values["dispersive_term_multipliers"])); - result.dispersive_term_rows = - index_vector(exact_array(values["dispersive_term_rows"])); - result.dispersive_term_channels = - index_vector(exact_array(values["dispersive_term_channels"])); - result.dispersive_coefficients = - real_vector(exact_array(values["dispersive_coefficients"])); - result.dispersive_pair_constants = - real_vector(exact_array(values["dispersive_pair_constants"])); - result.dispersive_group_offsets = - index_vector(exact_array(values["dispersive_group_offsets"])); - result.dispersive_group_members = - index_vector(exact_array(values["dispersive_group_members"])); - result.dispersive_quartics = - real_vector(exact_array(values["dispersive_quartics"])); - result.dispersive_input_scale = - real_vector(exact_array(values["dispersive_input_scale"])); - result.dispersive_output_scale = - real_vector(exact_array(values["dispersive_output_scale"])); - // The general modal route's group keys and node regions; absent elsewhere. - if (values.contains("dispersive_group_keys")) - result.dispersive_group_keys = - index_vector(exact_array(values["dispersive_group_keys"])); - if (values.contains("dispersive_node_region")) - result.dispersive_node_region = - index_vector(exact_array(values["dispersive_node_region"])); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("dispersive response data have wrong types"); - } - } - if (values.contains("general_modal")) { - try { - result.general_modal = nb::cast(values["general_modal"]); - if (result.general_modal) { - result.general_real_form = nb::cast(values["general_real_form"]); - result.general_map_intervals = - nb::cast(values["general_map_intervals"]); - result.general_map_n_deg = nb::cast(values["general_map_n_deg"]); - result.general_components = nb::cast(values["general_components"]); - result.general_nodes = nb::cast(values["general_nodes"]); - result.general_node_chunk = nb::cast(values["general_node_chunk"]); - result.general_responses = nb::cast(values["general_responses"]); - result.general_terms = nb::cast(values["general_terms"]); - result.general_pad_scale = nb::cast(values["general_pad_scale"]); - result.general_epsilon0 = nb::cast(values["general_epsilon0"]); - result.general_synthesis = - complex_vector(exact_array(values["general_synthesis"])); - result.general_projection = - complex_vector(exact_array(values["general_projection"])); - result.general_tensor = - real_vector(exact_array(values["general_tensor"])); - result.general_frequency_factor = complex_vector( - exact_array(values["general_frequency_factor"])); - result.general_delayed_tensor = - real_vector(exact_array(values["general_delayed_tensor"])); - result.general_delayed_spectrum = complex_vector( - exact_array(values["general_delayed_spectrum"])); - result.general_node_response = - index_vector(exact_array(values["general_node_response"])); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("general modal data have wrong types"); - } - } - if (values.contains("coupled_flow")) { - try { - result.coupled_flow = nb::cast(values["coupled_flow"]); - if (result.coupled_flow) { - result.flow_active = nb::cast(values["flow_active"]); - result.flow_sections = nb::cast(values["flow_sections"]); - result.flow_terms = nb::cast(values["flow_terms"]); - result.flow_slot = index_vector(exact_array(values["flow_slot"])); - result.flow_frozen = - complex_vector(exact_array(values["flow_frozen"])); - result.flow_residual_terms = - complex_vector(exact_array(values["flow_residual_terms"])); - if (values.contains("flow_left")) { - result.flow_left = - complex_vector(exact_array(values["flow_left"])); - result.flow_right = - complex_vector(exact_array(values["flow_right"])); - result.flow_eigenvalues = - complex_vector(exact_array(values["flow_eigenvalues"])); - result.flow_route = - index_vector(exact_array(values["flow_route"])); - result.flow_fallback_vectors = complex_vector( - exact_array(values["flow_fallback_vectors"])); - result.flow_fallback_triangular = complex_vector( - exact_array(values["flow_fallback_triangular"])); - } - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("coupled flow data have wrong types"); - } - } - return result; + "a native problem's cpu_execution must be a dict"); + self.cpu_execution = cpu_execution_from(nb::borrow(values)); + }, + nb::for_getter(nb::sig("def cpu_execution(self) -> dict[str, int | bool]")), + nb::for_setter(nb::sig( + "def cpu_execution(self, arg: dict[str, int | bool], /) -> None")), + nb::for_setter(nb::arg("values").none())) + .def_prop_rw( + "fft_planning", + [](const P& self) { return fft_request_dict(self.fft_planning); }, + [](P& self, const nb::handle values) { + if (!nb::isinstance(values)) + throw std::invalid_argument( + "a native problem's fft_planning must be a dict"); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + self.fft_planning = fft_planning_from(nb::borrow(values)); +#else + // A device build plans its transforms with the vendor's FFT alone. + static_cast(self); + static_cast(values); +#endif + }, + nb::for_getter(nb::sig("def fft_planning(self) -> dict[str, object]")), + nb::for_setter( + nb::sig("def fft_planning(self, arg: dict[str, object], /) -> None")), + nb::for_setter(nb::arg("values").none())) + .def("validate", &P::validate); } native::TrialScalars trial_from(const RealInput phi_delta, const RealInput phi_at, const RealInput prefactor, const RealInput density, const RealInput taper, const RealInput preionisation, const RealInput raman_rates) { - return {real_vector(phi_delta), - real_vector(phi_at), - real_vector(prefactor), - real_vector(density), - real_vector(taper), - real_vector(preionisation), - real_vector(raman_rates), - {}, - {}, - {}, - {}, - 0, - {}, - {}, - {}}; + return {.phi_delta = real_vector(phi_delta), + .phi_at = real_vector(phi_at), + .prefactor = real_vector(prefactor), + .density = real_vector(density), + .taper = real_vector(taper), + .preionisation = real_vector(preionisation), + .raman_rates = real_vector(raman_rates), + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; } // A trial's scalars with its material rows, as the Adams entry points take them. @@ -837,8 +815,8 @@ native::stage::Tables stage_tables_from(const nb::dict values) { const int chord = nb::cast(values["chord"]); if (chord < static_cast(native::stage::Chord::none) || chord > static_cast(native::stage::Chord::table)) - throw std::invalid_argument("the stage tables are inconsistent: chord kind " + - std::to_string(chord) + " is unknown"); + native::refuse("the stage tables are inconsistent: chord kind ", chord, + " is unknown"); tables.chord = static_cast(chord); tables.chord_rate = nb::cast(values["chord_rate"]); tables.chord_terms = nb::cast>(values["chord_terms"]); @@ -876,6 +854,7 @@ native::stage::Tables stage_tables_from(const nb::dict values) { (tables.flow.present() && tables.flow.edges.size() < 2) || (tables.transform.present() && tables.transform.edges.size() < 2)) throw std::invalid_argument("the stage tables are inconsistent"); + native::stage::require_scalar_tables(tables); return tables; } @@ -958,8 +937,8 @@ ComplexOutput complex_output(const std::vector& values, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); - return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); + static_cast(storage.release()); // the capsule owns it now + return {pointer->data(), shape.size(), shape.data(), owner}; } /** A complex array owning `storage`. */ @@ -969,8 +948,8 @@ ComplexOutput owned_complex(std::unique_ptr storage, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); - return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); + static_cast(storage.release()); // the capsule owns it now + return {pointer->data(), shape.size(), shape.data(), owner}; } /** A complex array owning `storage`, uninitialised doubles a native copy filled. */ @@ -980,8 +959,8 @@ ComplexOutput complex_buffer_output(native::crossing::HostBuffer storage, nb::capsule owner(pointer, [](void* value) noexcept { const native::crossing::HostBuffer held(static_cast(value)); }); - return ComplexOutput(reinterpret_cast*>(pointer), shape.size(), - shape.data(), owner); + return {reinterpret_cast*>(pointer), shape.size(), shape.data(), + owner}; } RealOutput real_output(std::vector values, const std::size_t rows) { @@ -992,9 +971,9 @@ RealOutput real_output(std::vector values, const std::size_t rows) { nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{rows, pointer->size() / rows}; - return RealOutput(pointer->data(), shape.size(), shape.data(), owner); + return {pointer->data(), shape.size(), shape.data(), owner}; } void initialize_kokkos(const int threads, const int kernels, const int fft, @@ -1014,8 +993,11 @@ void initialize_kokkos(const int threads, const int kernels, const int fft, throw std::runtime_error( "Kokkos was already initialized with another thread count"); } - native::configure_thread_policy( - {kernels, fft, raman_fft, raman_tail_fft, band_tiles}); + native::configure_thread_policy({.kernels = kernels, + .fft = fft, + .raman_fft = raman_fft, + .raman_tail_fft = raman_tail_fft, + .band_tiles = band_tiles}); } // ------------------------------------------------------------- the native Adams march @@ -1402,10 +1384,11 @@ march::State march_state_from(const nb::dict values) { state.cursor = nb::cast(values["cursor"]); for (const auto entry : nb::cast(values["history"])) { const auto parts = nb::cast(entry); - state.history.push_back({nb::cast(parts[0]), - nb::cast(parts[1]) ? march::Identity::predicted - : march::Identity::accepted, - nb::cast(parts[2])}); + state.history.push_back({.z = nb::cast(parts[0]), + .identity = nb::cast(parts[1]) + ? march::Identity::predicted + : march::Identity::accepted, + .row = nb::cast(parts[2])}); } const auto ledger = nb::cast(values["ledger"]); state.ledger.startup = nb::cast(ledger["startup"]); @@ -1532,7 +1515,8 @@ class ScriptedMarch { ScriptedMarch(const nb::dict policy, const RealInput norms, const double carrier_rate, const RealInput steps, const RealInput edges) : policy_(march_policy_from(policy)), norms_(real_vector(norms)), - carrier_rate_(carrier_rate), schedule_{real_vector(steps), real_vector(edges)} { + carrier_rate_(carrier_rate), + schedule_{.steps = real_vector(steps), .edges = real_vector(edges)} { if (norms_.size() % 3 != 0) throw std::invalid_argument("a script of norms is (attempts, 3)"); } @@ -1548,7 +1532,7 @@ class ScriptedMarch { } // What the march asked, one dict an entry: a startup's position, or a trial's // position, step, nodes, rows and orders with the row its accept freed. - nb::list log() const { + [[nodiscard]] nb::list log() const { nb::list out; for (const auto& entry : log_) { nb::dict event; @@ -1584,7 +1568,7 @@ class ScriptedRkMarch { ScriptedRkMarch(const nb::dict policy, const RealInput norms, const double h0, const RealInput steps, const RealInput edges) : policy_(rk_policy_from(policy)), norms_(real_vector(norms)), h0_(h0), - schedule_{real_vector(steps), real_vector(edges)} {} + schedule_{.steps = real_vector(steps), .edges = real_vector(edges)} {} nb::dict run(const nb::dict state_values, const nb::dict bounds_values) { auto state = rk_state_from(state_values); const auto bounds = march_bounds_from(bounds_values); @@ -1593,7 +1577,7 @@ class ScriptedRkMarch { return rk_result_to(result, state); } // Each trial's position and step, one dict a trial. - nb::list log() const { + [[nodiscard]] nb::list log() const { nb::list out; for (const auto& entry : log_) { nb::dict event; @@ -1616,11 +1600,12 @@ class ScriptedRkMarch { class BoundState { public: - explicit BoundState(const nb::dict& problem) : BoundState(problem_from(problem)) {} explicit BoundState(native::Problem problem) - : columns_(problem.n_columns()), elements_(problem.n_columns() * problem.n_omega), - stages_(problem.n_stages), raman_terms_(problem.n_raman_terms), - raman_(problem.raman) { + : columns_(problem.n_columns()), + elements_(problem.n_columns() * problem.dims.n_omega), + stages_(problem.stepper.n_stages), raman_terms_(problem.raman.n_terms), + raman_(problem.raman.enabled) { + problem.validate(); general_state_ = std::make_shared(std::move(problem)); const std::scoped_lock lock(runtime_state.mutex); runtime_state.states.push_back(this); @@ -1628,13 +1613,15 @@ class BoundState { ~BoundState() { close(); const std::scoped_lock lock(runtime_state.mutex); - const auto found = - std::find(runtime_state.states.begin(), runtime_state.states.end(), this); + const auto found = std::ranges::find(runtime_state.states, this); if (found != runtime_state.states.end()) runtime_state.states.erase(found); } BoundState(const BoundState&) = delete; BoundState& operator=(const BoundState&) = delete; + // The runtime's list of live states holds this address. + BoundState(BoundState&&) = delete; + BoundState& operator=(BoundState&&) = delete; void set_initial(const std::span field, const std::span noise) { @@ -1682,8 +1669,11 @@ class BoundState { const std::size_t sections = row_sum.size(); if (sections == 0 || condition_bound.size() % sections != 0) throw std::invalid_argument("the flow guard's certificate is one row a section"); - flow_guard_ = FlowGuard{std::move(row_sum), std::move(condition_bound), - taylor_scale, max_squarings, allowance}; + flow_guard_ = FlowGuard{.row_sum = std::move(row_sum), + .condition_bound = std::move(condition_bound), + .taylor_scale = taylor_scale, + .max_squarings = max_squarings, + .allowance = allowance}; flow_key_.reset(); } void guarded_flow_actions(const std::size_t section, const double dz, @@ -1708,12 +1698,11 @@ class BoundState { squarings = static_cast(std::ceil(std::log2(largest / guard.taylor_scale))); if (squarings > guard.max_squarings) - throw native::BackendRefusal( + native::refuse( "native cannot take a coupled flow action whose exponent has row-sum " - "norm " + - py_g(largest, 4) + ": it would need " + std::to_string(squarings) + - " squarings, above the " + std::to_string(guard.max_squarings) + - " this backend admits. Cap dz_max"); + "norm ", + py_g(largest, 4), ": it would need ", squarings, " squarings, above the ", + guard.max_squarings, " this backend admits. Cap dz_max"); } const auto measured = general_state_->form_flow_actions(section, dz, fractions); const std::size_t width = guard.condition_bound.size() / guard.row_sum.size(); @@ -1732,13 +1721,12 @@ class BoundState { } } if (largest > 1.0) - throw native::BackendRefusal( - "native measured a coupled flow conditioning of " + - py_g(measured[worst], 12) + " over a step of " + py_g(dz, 4) + - " m, above the " + py_g(declared[worst], 12) + - " the certificate declares at active bin " + std::to_string(worst) + - " of section " + std::to_string(section) + " -- exceeding it by a factor " + - py_g(largest, 12) + + native::refuse( + "native measured a coupled flow conditioning of ", + py_g(measured[worst], 12), " over a step of ", py_g(dz, 4), + " m, above the ", py_g(declared[worst], 12), + " the certificate declares at active bin ", worst, " of section ", section, + " -- exceeding it by a factor ", py_g(largest, 12), " of the rounding allowance: the inverse action carries the nonlinear " "source back by the measured factor. Lower dz_max, which is what the " "bound was declared over"); @@ -1758,7 +1746,7 @@ class BoundState { rk_nodes_ = std::move(nodes); rk_rtol_ = rtol; rk_atol_ = atol; - rk_schedule_ = {std::move(steps), std::move(edges)}; + rk_schedule_ = {.steps = std::move(steps), .edges = std::move(edges)}; } nb::dict rk_march_run(const nb::dict state_values, const nb::dict bounds_values) { require_unleased(); @@ -1776,7 +1764,7 @@ class BoundState { native::rk::Result result; { const nb::gil_scoped_release release; - result = native::rk::run(*rk_policy_, state, engine, bounds, rk_schedule_); + result = native::rk::run(rk_policy_.value(), state, engine, bounds, rk_schedule_); } return rk_result_to(result, state); } @@ -1825,7 +1813,8 @@ class BoundState { open_state(); if (flows > 2) throw std::invalid_argument("the low-storage march holds at most two flows"); - low_storage_ = native::rk::low_storage::Config{flows, separable, origin}; + low_storage_ = native::rk::low_storage::Config{ + .flows = flows, .separable = separable, .origin = origin}; } // solprop SEP 45 phase 4: a weighted controller's weights, and its norm from a // trial's partials, for both Runge-Kutta loops. @@ -1918,7 +1907,7 @@ class BoundState { general_state_->field_to_host(destination); } /** The state's element count, `columns * n_omega`. */ - std::size_t elements() const noexcept { return elements_; } + [[nodiscard]] std::size_t elements() const noexcept { return elements_; } std::vector adams_source(const std::vector& input, const native::TrialScalars& scalars) { require_open(); @@ -1975,9 +1964,9 @@ class BoundState { const auto row = native::stage::row(*stage_tables_, stage_anchor_, z0, z1); auto step = native::stage::trial(*stage_tables_, row); auto sample = native::stage::sample(*stage_tables_, stage_anchor_, step, z1); - return {std::move(step), std::move(sample)}; + return {.step = std::move(step), .sample = std::move(sample)}; } - std::vector stage_rate(const double z) const { + [[nodiscard]] std::vector stage_rate(const double z) const { require_open(); if (!stage_tables_) throw std::logic_error("no stage tables were uploaded"); @@ -2004,13 +1993,14 @@ class BoundState { auto state = march_state_from(state_values); const auto bounds = march_bounds_from(bounds_values); auto& general = open_state(); - native::engines::ResidentEngine engine(general, *stage_tables_, stage_anchor_, - march_fused_, march_compensated_, - march_interpolate_, march_fault_); + native::engines::ResidentEngine engine( + general, stage_tables_.value(), stage_anchor_, march_fused_, march_compensated_, + march_interpolate_, march_fault_); march::Result result; { const nb::gil_scoped_release release; - result = march::run(*march_policy_, state, engine, bounds, march_schedule_); + result = + march::run(march_policy_.value(), state, engine, bounds, march_schedule_); } return march_result_to(result, state); } @@ -2052,7 +2042,7 @@ class BoundState { return general_state_->projected_to_host(coarse); } // Not `require_open`: a closed state, whose pointer is null, reads as having none. - std::map dispersive_path() const { + [[nodiscard]] std::map dispersive_path() const { return general_state_ ? general_state_->dispersive_path() : std::map{}; } @@ -2096,51 +2086,51 @@ class BoundState { require_built(); return general_state_->memory_ledger(); } - native::CpuSizing execution_sizing() const { + [[nodiscard]] native::CpuSizing execution_sizing() const { require_built(); return general_state_->execution_sizing(); } - nb::dict cpu_execution() const { + [[nodiscard]] nb::dict cpu_execution() const { require_built(); return cpu_execution_dict(general_state_->cpu_execution()); } - nb::dict fft_planning() const { + [[nodiscard]] nb::dict fft_planning() const { require_built(); return fft_planning_dict(general_state_->fft_planning()); } - double planning_seconds() const { + [[nodiscard]] double planning_seconds() const { require_built(); return general_state_->planning_seconds(); } - double raman_planning_seconds() const { + [[nodiscard]] double raman_planning_seconds() const { require_built(); return general_state_->raman_planning_seconds(); } - std::string raman_fft_provider() const { + [[nodiscard]] std::string raman_fft_provider() const { require_built(); return general_state_->raman_fft_provider(); } - std::map raman_fft_tasks() const { + [[nodiscard]] std::map raman_fft_tasks() const { require_built(); return general_state_->raman_fft_tasks(); } - std::size_t raman_kernel_transforms() const { + [[nodiscard]] std::size_t raman_kernel_transforms() const { require_built(); return general_state_->raman_kernel_transforms(); } - bool fsal_ready() const { + [[nodiscard]] bool fsal_ready() const { require_built(); return general_state_->fsal_ready(); } - std::size_t rk_source_evaluations() const { + [[nodiscard]] std::size_t rk_source_evaluations() const { require_built(); return general_state_->rk_source_evaluations(); } - std::size_t axis_scans() const { + [[nodiscard]] std::size_t axis_scans() const { require_built(); return general_state_->axis_scans(); } - double quadrature_error() const { + [[nodiscard]] double quadrature_error() const { require_open(); return general_state_->quadrature_error(); } @@ -2148,27 +2138,29 @@ class BoundState { require_open(); return general_state_->quadrature_peak(); } - std::string synthesis_provider() const { + [[nodiscard]] std::string synthesis_provider() const { require_built(); return general_state_->synthesis_provider(); } - std::string projection_provider() const { + [[nodiscard]] std::string projection_provider() const { require_built(); return general_state_->projection_provider(); } - std::string gemm_shape_policy() const { + [[nodiscard]] std::string gemm_shape_policy() const { require_built(); return general_state_->gemm_shape_policy(); } - std::size_t response_team_size() const { + [[nodiscard]] std::size_t response_team_size() const { require_built(); return general_state_->response_team_size(); } - std::size_t columns() const noexcept { return columns_; } - std::size_t stages() const noexcept { return stages_; } - std::size_t raman_terms() const noexcept { return raman_terms_; } - bool raman() const noexcept { return raman_; } - bool closed() const noexcept { return !general_state_ || general_state_->closed(); } + [[nodiscard]] std::size_t columns() const noexcept { return columns_; } + [[nodiscard]] std::size_t stages() const noexcept { return stages_; } + [[nodiscard]] std::size_t raman_terms() const noexcept { return raman_terms_; } + [[nodiscard]] bool raman() const noexcept { return raman_; } + [[nodiscard]] bool closed() const noexcept { + return !general_state_ || general_state_->closed(); + } // Closes a session that leases this state first (below, after its class). void close() noexcept; @@ -2247,7 +2239,7 @@ class BoundState { return method; } throw std::invalid_argument( - "a session's march is \"adams\", \"runge_kutta\" or \"low_storage\""); + R"(a session's march is "adams", "runge_kutta" or "low_storage")"); } // The state's field and statistics, as a session reads them. std::unique_ptr session_resident() { @@ -2294,7 +2286,7 @@ class BoundState { if (closed()) throw std::logic_error("native state is closed"); } - native::GeneralState& built_state() const { + [[nodiscard]] native::GeneralState& built_state() const { require_built(); return *general_state_; } @@ -2352,7 +2344,7 @@ moved_array(std::vector&& values, const std::size_t rows, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr> held(static_cast*>(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{rows, columns}; return {pointer->data(), shape.size(), shape.data(), owner}; } @@ -2392,6 +2384,8 @@ class BoundSession { ~BoundSession() { close(); } BoundSession(const BoundSession&) = delete; BoundSession& operator=(const BoundSession&) = delete; + BoundSession(BoundSession&&) = delete; + BoundSession& operator=(BoundSession&&) = delete; // The advance runs without the interpreter lock; its batch becomes a record with it. BatchRecord advance(const std::size_t work_limit, const double deadline) { @@ -2411,13 +2405,11 @@ class BoundSession { nb::ndarray, nb::c_contig, nb::device::cpu> out) { const std::size_t n = session_->elements(); if (out.size() != n) - throw std::invalid_argument("an export destination has " + - std::to_string(out.size()) + " elements, expected " + - std::to_string(n)); + native::refuse("an export destination has ", out.size(), " elements, expected ", + n); if (weight && weight->size() != n) - throw std::invalid_argument("an export's power weight has " + - std::to_string(weight->size()) + - " elements, expected " + std::to_string(n)); + native::refuse("an export's power weight has ", weight->size(), + " elements, expected ", n); const double* values = weight ? weight->data() : nullptr; auto* destination = reinterpret_cast(out.data()); const nb::gil_scoped_release release; @@ -2428,8 +2420,10 @@ class BoundSession { const auto receipt = session_->request_snapshot(); return nb::make_tuple(receipt.request, receipt.coalesced); } - native::session::Status status() const noexcept { return session_->status(); } - std::size_t elements() const { return session_->elements(); } + [[nodiscard]] native::session::Status status() const noexcept { + return session_->status(); + } + [[nodiscard]] std::size_t elements() const { return session_->elements(); } // Continue a resumed run's monitor record; takes the session's lock. void continue_monitor(const nb::dict record, const bool plane, const nb::handle unwatched, const nb::dict edge_record) { @@ -2825,7 +2819,7 @@ std::string decode_record_text(std::string value) { return value; } -#if defined(_WIN32) +#ifdef _WIN32 // Windows looks a symbol up in one module only, so the search walks the modules this // extension imports (its FFT, BLAS and threading libraries, under whatever names the // wheel repair gave them), as dlsym on the extension's own handle does on POSIX. @@ -2968,8 +2962,8 @@ nb::dict build_record() { result["kokkos_fft_version"] = SOLPROP_NATIVE_KOKKOSFFT_VERSION; result["kokkos_kernels_version"] = SOLPROP_NATIVE_KOKKOSKERNELS_VERSION; result["nanobind_version"] = SOLPROP_NATIVE_NANOBIND_VERSION; - result["gemm_provider"] = SOLPROP_NATIVE_GEMM_PROVIDER; - result["tpl_blas_enabled"] = static_cast(SOLPROP_NATIVE_TPL_BLAS_ENABLED); + result["gemm_provider"] = "HYBRID"; + result["tpl_blas_enabled"] = (SOLPROP_NATIVE_TPL_BLAS_ENABLED != 0); result["blas_backend"] = SOLPROP_NATIVE_BLAS_BACKEND; result["blas_thread_policy"] = decode_record_text(SOLPROP_NATIVE_BLAS_THREAD_POLICY); result["blas_selection"] = decode_record_text(SOLPROP_NATIVE_BLAS_SELECTION); @@ -3002,7 +2996,7 @@ nb::dict build_record() { result["blas_threads"] = blas_threads; result["blas_threading"] = blas_threading; result["execution_space"] = native::ExecutionSpace::name(); -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA result["fft_backend"] = "cuFFT"; result["fft_library"] = loaded_library("cufftExecZ2Z"); result["blas_library"] = loaded_library("cublasDgemm_v2"); @@ -3032,7 +3026,7 @@ nb::dict build_record() { result["device_count"] = 0; #endif result["raman_fft_option"] = SOLPROP_NATIVE_RAMAN_FFT_PROVIDER; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE result["raman_fft_library"] = loaded_library("vDSP_fftm_zropD"); #else result["raman_fft_library"] = result["fft_library"]; @@ -3040,7 +3034,7 @@ nb::dict build_record() { result["openmp_library"] = loaded_library("omp_get_num_threads"); result["device_team_size"] = SOLPROP_NATIVE_DEVICE_TEAM_SIZE_RECORD; result["device_reduction_size"] = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE_RECORD; -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP result["openmp_enabled"] = true; #else result["openmp_enabled"] = false; @@ -3067,11 +3061,10 @@ nb::dict build_record() { result["compiler"] = __VERSION__; result["compiler_flags"] = decode_record_text(SOLPROP_NATIVE_COMPILER_FLAGS); result["build_type"] = SOLPROP_NATIVE_BUILD_TYPE; - result["coverage_instrumented"] = - static_cast(SOLPROP_NATIVE_COVERAGE_INSTRUMENTED); + result["coverage_instrumented"] = (SOLPROP_NATIVE_COVERAGE_INSTRUMENTED != 0); result["coverage_source_hash"] = SOLPROP_NATIVE_COVERAGE_SOURCE_HASH; result["fft_normalization"] = "none (explicit inverse scaling)"; -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA result["fft_planner"] = "Kokkos-FFT persistent Plan / cuFFT"; #elif defined(KOKKOS_ENABLE_HIP) result["fft_planner"] = "Kokkos-FFT persistent Plan / rocFFT"; @@ -3083,10 +3076,10 @@ nb::dict build_record() { "SOLPROP_FFTW_PLAN overrides"; #endif result["layout"] = "LayoutRight"; - result["native_arch_enabled"] = static_cast(SOLPROP_NATIVE_HOST_ARCH_ENABLED); + result["native_arch_enabled"] = (SOLPROP_NATIVE_HOST_ARCH_ENABLED != 0); result["aggressive_vectorization_enabled"] = - static_cast(SOLPROP_NATIVE_AGGRESSIVE_VECTORIZATION_ENABLED); - result["ipo_enabled"] = static_cast(SOLPROP_NATIVE_IPO_ENABLED); + (SOLPROP_NATIVE_AGGRESSIVE_VECTORIZATION_ENABLED != 0); + result["ipo_enabled"] = (SOLPROP_NATIVE_IPO_ENABLED != 0); const std::string cpu_baseline = SOLPROP_NATIVE_CPU_BASELINE; const std::string architecture_policy = SOLPROP_NATIVE_HOST_ARCH_ENABLED ? "native architecture" @@ -3107,15 +3100,13 @@ nb::dict build_record() { "; no fast-math; " + contraction_policy + "; " + architecture_policy + "; " + ipo_policy + "; " + vectorization_policy; -#if defined(__FAST_MATH__) +#ifdef __FAST_MATH__ result["fast_math_enabled"] = true; #else result["fast_math_enabled"] = false; #endif -#if defined(__apple_build_version__) +#ifdef __apple_build_version__ result["compiler_family"] = "AppleClang"; -#elif defined(__INTEL_LLVM_COMPILER) - result["compiler_family"] = "IntelLLVM"; #elif defined(__clang__) result["compiler_family"] = "Clang"; #elif defined(__GNUC__) @@ -3130,8 +3121,11 @@ nb::dict build_record() { } // namespace NB_MODULE(_core, module_) { - nb::exception(module_, "EnvelopeConvergenceError"); - nb::exception(module_, "BackendRefusal"); + // Each registration makes the Python exception type; the handle need not outlive it. + const nb::exception envelope_convergence_error( + module_, "EnvelopeConvergenceError"); + const nb::exception backend_refusal(module_, + "BackendRefusal"); module_.doc() = "Private nanobind seam for solprop-native"; module_.def("initialize_runtime", &initialize_kokkos, "threads"_a, "kernels"_a = 0, "fft"_a = 0, "raman_fft"_a = 0, "raman_tail_fft"_a = 0, @@ -3254,7 +3248,7 @@ NB_MODULE(_core, module_) { }, "policy"_a, "occupancy"_a); module_.def("available_device_memory", []() -> nb::object { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA std::size_t free_bytes = 0; std::size_t total_bytes = 0; if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) @@ -3341,6 +3335,7 @@ NB_MODULE(_core, module_) { }, "text"_a); native::bindings::bind_helpers(module_); + bind_problem(module_); native::bindings::bind_transverse(module_); nb::class_(module_, "ScriptedMarch") .def(nb::init(), "policy"_a, @@ -3355,7 +3350,7 @@ NB_MODULE(_core, module_) { .def("log", &ScriptedRkMarch::log) .def("used", &ScriptedRkMarch::used); nb::class_(module_, "State") - .def(nb::init()) + .def(nb::init(), "problem"_a) .def( "set_initial", [](BoundState& self, const ComplexInput field, const ComplexInput noise) { @@ -3531,7 +3526,7 @@ NB_MODULE(_core, module_) { const std::unique_ptr held( static_cast(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{extents[0], extents[1], extents[2]}; return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); }, @@ -3544,10 +3539,8 @@ NB_MODULE(_core, module_) { const auto extents = state.adams_row_extents(name); if (rows.ndim() != 3 || rows.shape(0) != extents[0] || rows.shape(1) != extents[1] || rows.shape(2) != extents[2]) - throw std::invalid_argument( - "the Adams history row '" + name + "' must have shape (" + - std::to_string(extents[0]) + ", " + std::to_string(extents[1]) + - ", " + std::to_string(extents[2]) + ")"); + native::refuse("the Adams history row '", name, "' must have shape (", + extents[0], ", ", extents[1], ", ", extents[2], ")"); const auto values = complex_vector(rows); const nb::gil_scoped_release release; state.adams_rows_from_host(name, values); @@ -3744,8 +3737,9 @@ NB_MODULE(_core, module_) { [](BoundState& self, const nb::dict policy, const bool fused, const bool compensated, const bool interpolate, const RealInput steps, const RealInput edges) { - self.march_configure(march_policy_from(policy), fused, compensated, - interpolate, {real_vector(steps), real_vector(edges)}); + self.march_configure( + march_policy_from(policy), fused, compensated, interpolate, + {.steps = real_vector(steps), .edges = real_vector(edges)}); }, "policy"_a, "fused"_a, "compensated"_a, "interpolate"_a, "steps"_a.noconvert(), "edges"_a.noconvert()) @@ -4281,8 +4275,8 @@ NB_MODULE(_core, module_) { .def_prop_ro("closed", &BoundState::closed) .def("close", &BoundState::close, nb::call_guard()); // solprop SEP 47: the native execution session and its compact event records. - nb::exception(module_, "SessionLifecycleError", - PyExc_RuntimeError); + const nb::exception lifecycle_error( + module_, "SessionLifecycleError", PyExc_RuntimeError); nb::class_(module_, "SessionEvent") .def_prop_ro("kind", [](const native::session::Event& event) { diff --git a/native/src/bindings_transverse.cpp b/native/src/bindings_transverse.cpp index 7d01aa2fb..52d66167b 100644 --- a/native/src/bindings_transverse.cpp +++ b/native/src/bindings_transverse.cpp @@ -248,10 +248,10 @@ void bind_transverse(nb::module_& module_) { "launch", [](transverse::Edge& self, const nb::dict reading, const double z, const std::optional> sums, - const std::optional noise) { + const std::optional& noise) { std::optional held; if (sums) - held = transverse::LaunchSums{sums->first, sums->second}; + held = transverse::LaunchSums{.total = sums->first, .ramp = sums->second}; std::optional carried; if (noise) carried = transverse_noise_from(*noise); @@ -266,7 +266,7 @@ void bind_transverse(nb::module_& module_) { const std::optional> sums) { std::optional held; if (sums) - held = transverse::ReadSums{sums->first, sums->second}; + held = transverse::ReadSums{.raw = sums->first, .noise = sums->second}; return transverse_notice_to( self.read(reading_from(reading), z, held ? &*held : nullptr)); }, diff --git a/native/src/core.cpp b/native/src/core.cpp index 37a685063..8e71bff9e 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -11,10 +11,11 @@ #include #include #include +#include #include #include -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP #include #endif @@ -30,7 +31,11 @@ ThreadPolicy thread_policy() { if (configured_threads.kernels != 0) return configured_threads; const int count = Kokkos::DefaultHostExecutionSpace().concurrency(); - return {count, count, count, count, -1}; + return {.kernels = count, + .fft = count, + .raman_fft = count, + .raman_tail_fft = count, + .band_tiles = -1}; } void configure_thread_policy(ThreadPolicy requested) { @@ -62,7 +67,7 @@ ExecutionSpace execution_instance(const int threads) { #else static_cast(threads); #endif - return ExecutionSpace(); + return {}; } void require_cpu_execution(const CpuExecution& execution) { @@ -198,7 +203,7 @@ namespace { // variable in a device build under -Werror). std::atomic openmp_blas{false}; thread_local int blas_holds = 0; -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP thread_local int blas_saved_default = 0; // The calling thread's scopes, and its OpenMP default before the outermost of them. thread_local int scope_depth = 0; @@ -218,10 +223,10 @@ thread_local int scope_saved_blocktime = -1; // OpenMP between that width and Kokkos' at every pair: 9 to 19 times slower on two // Intel Xeon Gold 6240 (solprop issue #130). void enter_runtime_default() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP if (scope_depth++ != 0) return; -#if defined(KMP_VERSION_MAJOR) +#ifdef KMP_VERSION_MAJOR // LLVM's runtime: idle threads spin for a short while inside native work and sleep // at once outside it (solprop ADR-071). scope_saved_blocktime = -1; @@ -244,12 +249,12 @@ void enter_runtime_default() noexcept { #endif } void leave_runtime_default() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP if (--scope_depth != 0) return; if (scope_saved_default != 0) omp_set_num_threads(scope_saved_default); -#if defined(KMP_VERSION_MAJOR) +#ifdef KMP_VERSION_MAJOR if (scope_saved_blocktime >= 0) kmp_set_blocktime(scope_saved_blocktime); #endif @@ -263,13 +268,13 @@ void leave_runtime_default() noexcept { // scope names its width (Kokkos's instances, `fftw_team_loop`), so the default reaches // only OpenBLAS. void hold_blas() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP blas_saved_default = omp_get_max_threads(); omp_set_num_threads(1); #endif } void release_blas() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP omp_set_num_threads(blas_saved_default); #endif } @@ -310,18 +315,51 @@ int scoped_block_time() noexcept { int scoped_openmp_default() noexcept { const CpuTeamScope scope(0); -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP return omp_get_max_threads(); #else return 1; #endif } +void Problem::validate() const { + const auto require = [](const bool holds, const char* message) { + if (!holds) + throw std::invalid_argument(message); + }; + require(cpu_execution != CpuExecution{}, + "a native problem needs its CPU execution record (cpu_execution, solprop " + "SEP 46)"); + if (raman.enabled) { + require(raman.n_terms != 0 && + dims.n_to <= + std::numeric_limits::max() / (2 * raman.n_terms) && + raman.undamped.size() == raman.n_terms * 2 * dims.n_to, + "raman_undamped must have shape (n_terms, 2 * n_to)"); + require(raman.tensor_weights.size() == 2 * raman.n_terms, + "raman_tensor_weights must have shape (n_terms, 2)"); + } else { + require(raman.n_terms == 0, + "raman.n_terms counts the terms of an enabled response"); + } + require(plasma.rate_coefficients.size() == 4 * plasma.rate_intervals, + "rate_coefficients must have shape (4, n_intervals)"); + if (plasma.envelope) { + constexpr std::size_t rule = std::size_t{4} * 512; + require(plasma.envelope_rule_nodes.size() == plasma.n_channels * rule && + plasma.envelope_rule_weights.size() == plasma.n_channels * rule, + "envelope rules require (n_channels, 4, 512)"); + require(plasma.envelope_drude_factor.size() == dims.n_omega && + plasma.envelope_loss_factor.size() == dims.n_omega, + "envelope current factors require (n_omega,)"); + } +} + bool compact_problem(const Problem& problem) noexcept { - return problem.n_columns() == 1 && problem.n_nodes == 1 && - problem.n_pol_transform == 1 && !problem.general_modal && - !problem.coupled_flow && !problem.separable_geometry() && - problem.n_to < problem.cpu_execution.compact_max_fine_samples; + return problem.n_columns() == 1 && problem.dims.n_nodes == 1 && + problem.dims.n_pol_transform == 1 && !problem.general.enabled && + !problem.flow.enabled && !problem.separable_geometry() && + problem.dims.n_to < problem.cpu_execution.compact_max_fine_samples; } const ExecutionSpace& CpuExecutionPool::get(const CpuOperation operation, diff --git a/native/src/coupled_flow.cpp b/native/src/coupled_flow.cpp index 71796578c..46d86dfd8 100644 --- a/native/src/coupled_flow.cpp +++ b/native/src/coupled_flow.cpp @@ -3,7 +3,9 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/coupled_flow.hpp" + #include "solprop_native/launch.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -11,6 +13,7 @@ #include #include #include +#include #include namespace solprop_native { @@ -25,13 +28,11 @@ std::size_t product(const std::size_t left, const std::size_t right, const char* void require_size(const std::size_t actual, const std::size_t expected, const char* name) { if (actual != expected) - throw std::invalid_argument(std::string("coupled flow ") + name + " has " + - std::to_string(actual) + " values, expected " + - std::to_string(expected)); + refuse("coupled flow ", name, " has ", actual, " values, expected ", expected); } bool finite(const std::vector& values) { - return std::all_of(values.begin(), values.end(), [](const Complex& value) { + return std::ranges::all_of(values, [](const Complex& value) { return std::isfinite(value.real()) && std::isfinite(value.imag()); }); } @@ -53,8 +54,8 @@ KOKKOS_INLINE_FUNCTION Complex flow_exponential(const Complex value) { std::size_t block_size(const std::size_t count, const Problem& problem, const char* name) { - return product(product(count, problem.flow_active, name), - product(problem.n_modes, problem.n_modes, name), name); + return product(product(count, problem.flow.active, name), + product(problem.dims.n_modes, problem.dims.n_modes, name), name); } using ComplexView3 = CoupledFlowAction::ComplexView3; @@ -261,26 +262,26 @@ namespace { std::size_t fallback_count(const Problem& problem) { return static_cast( - std::count_if(problem.flow_route.begin(), problem.flow_route.end(), + std::count_if(problem.flow.route.begin(), problem.flow.route.end(), [](const std::size_t value) { return value != 0; })); } void require_flow_eigenbasis(const Problem& problem) { - if (problem.flow_left.empty() && problem.flow_right.empty() && - problem.flow_eigenvalues.empty() && problem.flow_route.empty() && - problem.flow_fallback_vectors.empty() && problem.flow_fallback_triangular.empty()) + if (problem.flow.left.empty() && problem.flow.right.empty() && + problem.flow.eigenvalues.empty() && problem.flow.route.empty() && + problem.flow.fallback_vectors.empty() && problem.flow.fallback_triangular.empty()) return; - const auto sections = problem.flow_sections; - require_size(problem.flow_left.size(), block_size(sections, problem, "eigenbasis"), + const auto sections = problem.flow.sections; + require_size(problem.flow.left.size(), block_size(sections, problem, "eigenbasis"), "left eigenbasis"); - require_size(problem.flow_right.size(), block_size(sections, problem, "eigenbasis"), + require_size(problem.flow.right.size(), block_size(sections, problem, "eigenbasis"), "right eigenbasis"); - const auto bins = product(sections, problem.flow_active, "route"); - require_size(problem.flow_eigenvalues.size(), - product(bins, problem.n_modes, "eigenvalues"), "eigenvalues"); - require_size(problem.flow_route.size(), bins, "route"); + const auto bins = product(sections, problem.flow.active, "route"); + require_size(problem.flow.eigenvalues.size(), + product(bins, problem.dims.n_modes, "eigenvalues"), "eigenvalues"); + require_size(problem.flow.route.size(), bins, "route"); std::size_t expected = 1; - for (const auto route : problem.flow_route) { + for (const auto route : problem.flow.route) { if (route == 0) continue; if (route != expected) @@ -289,43 +290,44 @@ void require_flow_eigenbasis(const Problem& problem) { ++expected; } const auto fallback = expected - 1; - const auto blocks = product( - fallback, product(problem.n_modes, problem.n_modes, "fallback"), "fallback"); - require_size(problem.flow_fallback_vectors.size(), blocks, "fallback vectors"); - require_size(problem.flow_fallback_triangular.size(), blocks, "fallback triangular"); - if (!finite(problem.flow_left) || !finite(problem.flow_right) || - !finite(problem.flow_eigenvalues) || !finite(problem.flow_fallback_vectors) || - !finite(problem.flow_fallback_triangular)) + const auto blocks = + product(fallback, product(problem.dims.n_modes, problem.dims.n_modes, "fallback"), + "fallback"); + require_size(problem.flow.fallback_vectors.size(), blocks, "fallback vectors"); + require_size(problem.flow.fallback_triangular.size(), blocks, "fallback triangular"); + if (!finite(problem.flow.left) || !finite(problem.flow.right) || + !finite(problem.flow.eigenvalues) || !finite(problem.flow.fallback_vectors) || + !finite(problem.flow.fallback_triangular)) throw std::invalid_argument("coupled flow eigenbasis data must be finite"); } } // namespace void require_coupled_flow(const Problem& problem) { - if (!problem.coupled_flow) { - if (problem.flow_active != 0 || problem.flow_sections != 0 || - problem.flow_terms != 0 || !problem.flow_slot.empty() || - !problem.flow_frozen.empty() || !problem.flow_residual_terms.empty() || - !problem.flow_left.empty() || !problem.flow_route.empty()) + if (!problem.flow.enabled) { + if (problem.flow.active != 0 || problem.flow.sections != 0 || + problem.flow.terms != 0 || !problem.flow.slot.empty() || + !problem.flow.frozen.empty() || !problem.flow.residual_terms.empty() || + !problem.flow.left.empty() || !problem.flow.route.empty()) throw std::invalid_argument("coupled flow arrays require coupled_flow"); return; } // The action contracts the channel axis of a scalar state; the synthesis may resolve // two transverse components, which the flow never reads. - if (problem.n_pol_state != 1 || problem.separable_geometry()) + if (problem.dims.n_pol_state != 1 || problem.separable_geometry()) throw std::invalid_argument("a coupled flow mixes one channel axis of a scalar " "modal state, with no polarisation axis"); - if (problem.modified_noise) + if (problem.stepper.modified_noise) throw std::invalid_argument("a coupled flow carries no noise model"); - if (problem.n_stages == 0) + if (problem.stepper.n_stages == 0) throw std::invalid_argument("a coupled flow needs a tableau"); - if (problem.flow_active == 0 || problem.flow_active > problem.n_omega) + if (problem.flow.active == 0 || problem.flow.active > problem.dims.n_omega) throw std::invalid_argument("flow_active must lie in [1, n_omega]"); - if (problem.flow_sections == 0) + if (problem.flow.sections == 0) throw std::invalid_argument("a coupled flow has at least one section"); - require_size(problem.flow_slot.size(), problem.n_omega, "slot"); + require_size(problem.flow.slot.size(), problem.dims.n_omega, "slot"); std::size_t expected = 1; - for (const auto slot : problem.flow_slot) { + for (const auto slot : problem.flow.slot) { if (slot == 0) continue; if (slot != expected) @@ -333,34 +335,34 @@ void require_coupled_flow(const Problem& problem) { "in increasing frequency"); ++expected; } - if (expected - 1 != problem.flow_active) + if (expected - 1 != problem.flow.active) throw std::invalid_argument("flow_slot names a different number of active bins " "than flow_active"); - static_cast(block_size(problem.n_stages + 1, problem, "actions")); + static_cast(block_size(problem.stepper.n_stages + 1, problem, "actions")); require_flow_eigenbasis(problem); - if (problem.flow_terms == 0) { - if (!problem.flow_frozen.empty() || !problem.flow_residual_terms.empty()) + if (problem.flow.terms == 0) { + if (!problem.flow.frozen.empty() || !problem.flow.residual_terms.empty()) throw std::invalid_argument("a coupled flow with no explicit residual carries " "no frozen blocks or residual terms"); return; } - require_size(problem.flow_frozen.size(), - block_size(problem.flow_sections, problem, "frozen"), "frozen"); - require_size(problem.flow_residual_terms.size(), - block_size(problem.flow_terms, problem, "residual terms"), + require_size(problem.flow.frozen.size(), + block_size(problem.flow.sections, problem, "frozen"), "frozen"); + require_size(problem.flow.residual_terms.size(), + block_size(problem.flow.terms, problem, "residual terms"), "residual terms"); - if (!finite(problem.flow_frozen) || !finite(problem.flow_residual_terms)) + if (!finite(problem.flow.frozen) || !finite(problem.flow.residual_terms)) throw std::invalid_argument("coupled flow residual data must be finite"); } -CoupledFlowAction::CoupledFlowAction(const Problem& problem, - const ExecutionSpace& kernels) - : n_modes_(problem.n_modes), n_omega_(problem.n_omega), n_stages_(problem.n_stages), - active_(problem.flow_active), sections_(problem.flow_sections), - terms_(problem.flow_terms), identity_(problem.n_stages + 1, 0U), - listed_counts_(problem.n_stages, 0U), kernels_(kernels) { +CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kernels) + : n_modes_(problem.dims.n_modes), n_omega_(problem.dims.n_omega), + n_stages_(problem.stepper.n_stages), active_(problem.flow.active), + sections_(problem.flow.sections), terms_(problem.flow.terms), + identity_(problem.stepper.n_stages + 1, 0U), + listed_counts_(problem.stepper.n_stages, 0U), kernels_(std::move(kernels)) { require_coupled_flow(problem); - if (!problem.coupled_flow) + if (!problem.flow.enabled) throw std::invalid_argument("the problem does not declare a coupled flow"); const auto slots = n_stages_ + 1; slot_ = IndexView("solprop_native.flow.slot", n_omega_); @@ -386,11 +388,11 @@ CoupledFlowAction::CoupledFlowAction(const Problem& problem, host_active_terms_ = Kokkos::create_mirror_view(active_terms_); host_active_counts_ = Kokkos::create_mirror_view(active_counts_); lab_ = ComplexView2("solprop_native.flow.lab", n_modes_, n_omega_); - upload(kernels_, frozen_, problem.flow_frozen); - upload(kernels_, terms_view_, problem.flow_residual_terms); + upload(kernels_, frozen_, problem.flow.frozen); + upload(kernels_, terms_view_, problem.flow.residual_terms); } - upload(kernels_, slot_, problem.flow_slot); - forms_ = !problem.flow_left.empty(); + upload(kernels_, slot_, problem.flow.slot); + forms_ = !problem.flow.left.empty(); if (forms_) { fallback_ = fallback_count(problem); left_ = ComplexView4("solprop_native.flow.left", sections_, active_, n_modes_, @@ -413,16 +415,16 @@ CoupledFlowAction::CoupledFlowAction(const Problem& problem, host_fractions_ = Kokkos::create_mirror_view(fractions_); host_measured_ = Kokkos::create_mirror_view(measured_); measured_host_.assign(active_, 0.0); - upload(kernels_, left_, problem.flow_left); - upload(kernels_, right_, problem.flow_right); - upload(kernels_, eigenvalues_, problem.flow_eigenvalues); - upload(kernels_, route_, problem.flow_route); + upload(kernels_, left_, problem.flow.left); + upload(kernels_, right_, problem.flow.right); + upload(kernels_, eigenvalues_, problem.flow.eigenvalues); + upload(kernels_, route_, problem.flow.route); section_fallback_.assign(sections_, 0U); - for (std::size_t index = 0; index < problem.flow_route.size(); ++index) - if (problem.flow_route[index] != 0) + for (std::size_t index = 0; index < problem.flow.route.size(); ++index) + if (problem.flow.route[index] != 0) section_fallback_[index / active_] = 1U; - upload(kernels_, fallback_vectors_, problem.flow_fallback_vectors); - upload(kernels_, fallback_triangular_, problem.flow_fallback_triangular); + upload(kernels_, fallback_vectors_, problem.flow.fallback_vectors); + upload(kernels_, fallback_triangular_, problem.flow.fallback_triangular); } kernels_.fence("solprop_native coupled flow upload"); } @@ -439,7 +441,7 @@ CoupledFlowAction::form_actions(const std::size_t section, const double dz, throw std::invalid_argument("flow_section names no section of the coupled flow"); require_size(fractions.size(), n_stages_ + 1, "stage fractions"); if (!std::isfinite(dz) || dz <= 0.0 || - !std::all_of(fractions.begin(), fractions.end(), [](const double value) { + !std::ranges::all_of(fractions, [](const double value) { return std::isfinite(value) && value >= 0.0 && value <= 1.0; })) throw std::invalid_argument("coupled flow step and fractions must be finite, the " @@ -558,8 +560,8 @@ void CoupledFlowAction::set_trial(const std::size_t section, return; } require_size(weights.size(), n_stages_ * terms_, "flow_weights"); - if (!std::all_of(weights.begin(), weights.end(), - [](const double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(weights, + [](const double value) { return std::isfinite(value); })) throw std::invalid_argument("flow_weights must be finite"); upload(kernels_, weights_, weights); const auto& listed = host_active_terms_; diff --git a/native/src/crossing.cpp b/native/src/crossing.cpp index e84f9b5cc..c5ab49c6b 100644 --- a/native/src/crossing.cpp +++ b/native/src/crossing.cpp @@ -4,6 +4,8 @@ #include "solprop_native/crossing.hpp" +#include "solprop_native/build.hpp" + #include #include #include @@ -13,7 +15,7 @@ #include #include -#if defined(__linux__) +#ifdef __linux__ #include #include #endif @@ -21,16 +23,10 @@ namespace solprop_native::crossing { namespace { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -constexpr bool device_build = true; -#else -constexpr bool device_build = false; -#endif - // Kokkos holds its copies to one thread at a time per instance, so the workers call the // vendor's synchronous copy, which is safe from many threads: it is what was measured. void device_read(void* destination, const void* source, const std::size_t bytes) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaMemcpy(destination, source, bytes, cudaMemcpyDeviceToHost) != cudaSuccess) throw std::runtime_error("a state's copy to the host failed"); #elif defined(KOKKOS_ENABLE_HIP) @@ -43,7 +39,7 @@ void device_read(void* destination, const void* source, const std::size_t bytes) } void device_write(void* destination, const void* source, const std::size_t bytes) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaMemcpy(destination, source, bytes, cudaMemcpyHostToDevice) != cudaSuccess) throw std::runtime_error("a state's copy to the device failed"); #elif defined(KOKKOS_ENABLE_HIP) @@ -57,7 +53,7 @@ void device_write(void* destination, const void* source, const std::size_t bytes /** The device a new thread must select: the choice is per thread on both vendors. */ int current_device() { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA return Kokkos::Cuda{}.cuda_device(); #elif defined(KOKKOS_ENABLE_HIP) return Kokkos::HIP{}.hip_device(); @@ -67,7 +63,7 @@ int current_device() { } void select_device([[maybe_unused]] const int device) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaSetDevice(device) != cudaSuccess) throw std::runtime_error("a crossing's worker could not select its device"); #elif defined(KOKKOS_ENABLE_HIP) @@ -78,7 +74,7 @@ void select_device([[maybe_unused]] const int device) { /** The host threads this process may run on, at most `worker_limit`. */ std::size_t worker_count() { -#if defined(__linux__) +#ifdef __linux__ cpu_set_t allowed; CPU_ZERO(&allowed); if (sched_getaffinity(0, sizeof(allowed), &allowed) == 0) @@ -91,14 +87,14 @@ std::size_t worker_count() { } // namespace bool takes_ring(const std::size_t bytes) noexcept { - return device_build && bytes >= ring_threshold; + return build::device && bytes >= ring_threshold; } void BufferDeleter::operator()(double* pointer) const noexcept { std::free(pointer); } HostBuffer host_buffer(const std::size_t count) { const std::size_t bytes = count * sizeof(double); -#if defined(__linux__) +#ifdef __linux__ if (takes_ring(bytes)) { // The size huge pages come in, which a large buffer is aligned to. constexpr std::size_t huge_page = std::size_t{2} << 20; diff --git a/native/src/dense.cpp b/native/src/dense.cpp index 91db7bb82..fa2013310 100644 --- a/native/src/dense.cpp +++ b/native/src/dense.cpp @@ -3,14 +3,13 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/dense_output.hpp" +#include "solprop_native/engines.hpp" +#include "solprop_native/lagrange_basis.hpp" #include #include #include -#include "solprop_native/engines.hpp" -#include "solprop_native/lagrange_basis.hpp" - // As in session.cpp (solprop #135): the weights an observation reads round as // solprop's Python forms them, so this file takes no contraction (CMake) and the // optimize attribute this pragma gives every function below keeps GCC from inlining @@ -149,6 +148,7 @@ void RungeKuttaDense::stepped(const march::Result& /*result*/) {} bool RungeKuttaDense::crosses(const double position, const double target, const int ulps) const { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) return !march::reached(target, position, ulps); } diff --git a/native/src/dispersive.cpp b/native/src/dispersive.cpp index 1588b4041..1355ebac4 100644 --- a/native/src/dispersive.cpp +++ b/native/src/dispersive.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/dispersive.hpp" + #include "solprop_native/gemm.hpp" #include "solprop_native/launch.hpp" #include "solprop_native/slab.hpp" @@ -98,10 +99,12 @@ template struct SelfP x0 = x0 - offset[base]; x1 = x1 - offset[base + 1]; } - Complex p0, p1; + Complex p0; + Complex p1; dispersive_conserving(x0, x1, x0, x1, x0, x1, p0, p1); if constexpr (Form == DispersiveForm::harmonic) { - Complex h0, h1; + Complex h0; + Complex h1; dispersive_harmonic(x0, x1, x0, x1, x0, x1, carrier[time], h0, h1); p0 = p0 + h0; p1 = p1 + h1; @@ -155,10 +158,18 @@ struct PairProducts { if constexpr (Form == DispersiveForm::real) { if constexpr (Components == 2) { const std::size_t at1 = (base + 1) * n_to + time; - const double x0 = field[at0], x1 = field[at1]; - const double f0 = first[at0], f1 = first[at1]; - const double h0 = second[at0], h1 = second[at1]; - double p0 = 0.0, p1 = 0.0, a0 = 0.0, a1 = 0.0, b0 = 0.0, b1 = 0.0; + const double x0 = field[at0]; + const double x1 = field[at1]; + const double f0 = first[at0]; + const double f1 = first[at1]; + const double h0 = second[at0]; + const double h1 = second[at1]; + double p0 = 0.0; + double p1 = 0.0; + double a0 = 0.0; + double a1 = 0.0; + double b0 = 0.0; + double b1 = 0.0; dispersive_isotropic(x0, x1, x0, x1, f0, f1, p0, p1); dispersive_isotropic(x0, x1, x0, x1, h0, h1, a0, a1); dispersive_isotropic(x0, x1, f0, f1, f0, f1, b0, b1); @@ -167,7 +178,9 @@ struct PairProducts { second[at0] = (c0 * a0 + c1 * b0) * collar; second[at1] = (c0 * a1 + c1 * b1) * collar; } else { - const double x = field[at0], f = first[at0], h = second[at0]; + const double x = field[at0]; + const double f = first[at0]; + const double h = second[at0]; first[at0] = dispersive_real_pair(x, f) * collar; second[at0] = (c0 * dispersive_real_pair(x, h) + c1 * dispersive_real_cross(x, f)) * @@ -176,20 +189,29 @@ struct PairProducts { } else { if constexpr (Components == 2) { const std::size_t at1 = (base + 1) * n_to + time; - Complex x0 = field[at0], x1 = field[at1]; + Complex x0 = field[at0]; + Complex x1 = field[at1]; if (shifted) { x0 = x0 - offset[base]; x1 = x1 - offset[base + 1]; } - const Complex f0 = first[at0], f1 = first[at1]; - const Complex h0 = second[at0], h1 = second[at1]; - Complex p0, p1, a0, a1, b0, b1; + const Complex f0 = first[at0]; + const Complex f1 = first[at1]; + const Complex h0 = second[at0]; + const Complex h1 = second[at1]; + Complex p0; + Complex p1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; dispersive_conserving(x0, x1, x0, x1, f0, f1, p0, p1); dispersive_conserving(x0, x1, x0, x1, h0, h1, a0, a1); dispersive_conserving(x0, x1, f0, f1, f0, f1, b0, b1); if constexpr (Form == DispersiveForm::harmonic) { const Complex c = carrier[time]; - Complex g0, g1; + Complex g0; + Complex g1; dispersive_harmonic(x0, x1, x0, x1, f0, f1, c, g0, g1); p0 = p0 + g0; p1 = p1 + g1; @@ -505,7 +527,9 @@ template struct GeneralPairs { } return; } - Value x[Components], f[Components], h[Components]; + Value x[Components]; + Value f[Components]; + Value h[Components]; for (std::size_t c = 0; c < Components; ++c) { const std::size_t at = (base + c) * n_to + time; if constexpr (Real) { @@ -520,7 +544,9 @@ template struct GeneralPairs { h[c] = second_field[at]; } } - Value p[Components], a[Components], b[Components]; + Value p[Components]; + Value a[Components]; + Value b[Components]; if constexpr (Real) { dispersive_isotropic_n(x, x, f, p); dispersive_isotropic_n(x, x, h, a); @@ -548,50 +574,50 @@ template struct GeneralPairs { } // namespace void require_dispersive(const Problem& problem) { - if (!problem.dispersive) { - if (!problem.dispersive_inputs.empty() || - !problem.dispersive_coefficients.empty() || - !problem.dispersive_term_rows.empty() || - !problem.dispersive_group_offsets.empty() || - !problem.dispersive_group_keys.empty() || - !problem.dispersive_node_region.empty()) + if (!problem.dispersive.enabled) { + if (!problem.dispersive.inputs.empty() || + !problem.dispersive.coefficients.empty() || + !problem.dispersive.term_rows.empty() || + !problem.dispersive.group_offsets.empty() || + !problem.dispersive.group_keys.empty() || + !problem.dispersive.node_region.empty()) throw std::invalid_argument( "dispersive data were supplied while the dispersive response is disabled"); return; } - if (problem.dispersive_form < 0 || problem.dispersive_form > 2) + if (problem.dispersive.form < 0 || problem.dispersive.form > 2) throw std::invalid_argument("dispersive_form must be 0, 1 or 2"); - const auto form = static_cast(problem.dispersive_form); - const bool general = problem.general_modal; + const auto form = static_cast(problem.dispersive.form); + const bool general = problem.general.enabled; // The general route synthesises its own analytic signal on the real field family; // elsewhere the family follows the sector. - const bool family_fits = general - ? problem.family == FieldFamily::field_real && - form != DispersiveForm::harmonic - : (problem.family == FieldFamily::field_real && - form == DispersiveForm::real) || - (problem.family == FieldFamily::field_analytic && - form == DispersiveForm::conserving) || - (problem.family == FieldFamily::envelope && - form != DispersiveForm::real); + const bool family_fits = + general ? problem.dims.family == FieldFamily::field_real && + form != DispersiveForm::harmonic + : (problem.dims.family == FieldFamily::field_real && + form == DispersiveForm::real) || + (problem.dims.family == FieldFamily::field_analytic && + form == DispersiveForm::conserving) || + (problem.dims.family == FieldFamily::envelope && + form != DispersiveForm::real); if (!family_fits) throw std::invalid_argument( "the dispersive form does not fit the field family: a field grid's full cubic " "takes the real family, its number-conserving sector the analytic one (the " "general modal route's both the real one), and an envelope's both sectors the " "envelope family"); - const std::size_t bins = problem.family == FieldFamily::envelope && !general - ? problem.n_to - : problem.n_to / 2 + 1; - const std::size_t channels = problem.dispersive_coefficients.size(); + const std::size_t bins = problem.dims.family == FieldFamily::envelope && !general + ? problem.dims.n_to + : problem.dims.n_to / 2 + 1; + const std::size_t channels = problem.dispersive.coefficients.size(); if (channels == 0) throw std::invalid_argument("the dispersive response needs a channel"); - require_size(problem.dispersive_input_scale.size(), bins, "input_scale"); - require_size(problem.dispersive_output_scale.size(), bins, "output_scale"); - require_size(problem.dispersive_pair_constants.size(), + require_size(problem.dispersive.input_scale.size(), bins, "input_scale"); + require_size(problem.dispersive.output_scale.size(), bins, "output_scale"); + require_size(problem.dispersive.pair_constants.size(), product_of(2, channels, "pair constants"), "pair_constants"); - const auto& offsets = problem.dispersive_group_offsets; - const auto& members = problem.dispersive_group_members; + const auto& offsets = problem.dispersive.group_offsets; + const auto& members = problem.dispersive.group_members; if (offsets.size() < 2 || offsets.front() != 0 || offsets.back() != members.size() || members.size() != channels) throw std::invalid_argument( @@ -606,56 +632,57 @@ void require_dispersive(const Problem& problem) { for (std::size_t group = 0; group + 1 < offsets.size(); ++group) { if (offsets[group + 1] <= offsets[group]) throw std::invalid_argument("every dispersive group needs a channel"); - if (problem.dispersive_symmetric && offsets[group + 1] - offsets[group] > 1) + if (problem.dispersive.symmetric && offsets[group + 1] - offsets[group] > 1) ++mixtures; } const std::size_t groups = offsets.size() - 1; // Keys and node regions belong to the general route, where a channel acts at its own // region's nodes; elsewhere every channel acts at every node. - const auto& keys = problem.dispersive_group_keys; + const auto& keys = problem.dispersive.group_keys; if (general) { require_size(keys.size(), groups, "group_keys"); - require_size(problem.dispersive_node_region.size(), problem.general_nodes, + require_size(problem.dispersive.node_region.size(), problem.general.nodes, "node_region"); - } else if (!keys.empty() || !problem.dispersive_node_region.empty()) { + } else if (!keys.empty() || !problem.dispersive.node_region.empty()) { throw std::invalid_argument( "dispersive group keys and node regions belong to the general modal route"); } std::vector distinct; for (std::size_t group = 0; group < groups; ++group) { const auto key = keys.empty() ? 0 : keys[group]; - if (std::find(distinct.begin(), distinct.end(), key) == distinct.end()) + if (std::ranges::find(distinct, key) == distinct.end()) distinct.push_back(key); } - const std::size_t fields = problem.dispersive_symmetric ? groups : 2 * groups; + const std::size_t fields = problem.dispersive.symmetric ? groups : 2 * groups; const std::size_t products = - problem.dispersive_symmetric ? groups : distinct.size() + 2 * groups; - require_size(problem.dispersive_inputs.size(), product_of(fields, bins, "inputs"), + problem.dispersive.symmetric ? groups : distinct.size() + 2 * groups; + require_size(problem.dispersive.inputs.size(), product_of(fields, bins, "inputs"), "inputs"); - const std::size_t terms = problem.dispersive_term_rows.size(); + const std::size_t terms = problem.dispersive.term_rows.size(); if (terms == 0) throw std::invalid_argument("the dispersive response needs an output term"); - require_size(problem.dispersive_term_channels.size(), terms, "term_channels"); - require_size(problem.dispersive_term_multipliers.size(), + require_size(problem.dispersive.term_channels.size(), terms, "term_channels"); + require_size(problem.dispersive.term_multipliers.size(), product_of(terms, bins, "term multipliers"), "term_multipliers"); for (std::size_t term = 0; term < terms; ++term) - if (problem.dispersive_term_rows[term] >= products || - problem.dispersive_term_channels[term] > channels + mixtures) + if (problem.dispersive.term_rows[term] >= products || + problem.dispersive.term_channels[term] > channels + mixtures) throw std::invalid_argument("a dispersive term names no product or factor"); - require_size(problem.dispersive_quartics.size(), + require_size(problem.dispersive.quartics.size(), mixtures == 0 ? 0 : product_of(channels, bins, "quartics"), "quartics"); // The products and their spectra, and the node slots: the general route's chunk, or // at most the modal columns. - const auto rows = general ? product_of(problem.general_node_chunk, - problem.general_components, "dispersive rows") + const auto rows = general ? product_of(problem.general.node_chunk, + problem.general.components, "dispersive rows") : problem.n_columns(); const auto field = product_of(product_of(products + fields, rows, "dispersive products"), - problem.n_to, "dispersive products"); + problem.dims.n_to, "dispersive products"); static_cast(product_of(field, sizeof(Complex), "dispersive product bytes")); static_cast(product_of( - product_of(2 * problem.node_chunk, problem.effective_node_pol(), "slots"), - product_of(problem.n_to, sizeof(Complex), "slot bytes"), "dispersive slots")); + product_of(2 * problem.dims.node_chunk, problem.effective_node_pol(), "slots"), + product_of(problem.dims.n_to, sizeof(Complex), "slot bytes"), + "dispersive slots")); } void dispersive_mixture_rows( @@ -691,29 +718,28 @@ void dispersive_mixture_rows( // --- the layout and the stage's weights ---------------------------------------------- -DispersiveWeights::DispersiveWeights(const Problem& problem, - const ExecutionSpace& kernels) - : kernels_(kernels) { +DispersiveWeights::DispersiveWeights(const Problem& problem, ExecutionSpace kernels) + : kernels_(std::move(kernels)) { require_dispersive(problem); - if (!problem.dispersive) + if (!problem.dispersive.enabled) throw std::invalid_argument("the problem does not carry the dispersive response"); - channels_ = problem.dispersive_coefficients.size(); - bins_ = problem.dispersive_input_scale.size(); - n_stages_ = problem.n_stages; - symmetric_ = problem.dispersive_symmetric; - coefficients_ = problem.dispersive_coefficients; - pair_constants_ = problem.dispersive_pair_constants; - group_offsets_ = problem.dispersive_group_offsets; - group_members_ = problem.dispersive_group_members; + channels_ = problem.dispersive.coefficients.size(); + bins_ = problem.dispersive.input_scale.size(); + n_stages_ = problem.stepper.n_stages; + symmetric_ = problem.dispersive.symmetric; + coefficients_ = problem.dispersive.coefficients; + pair_constants_ = problem.dispersive.pair_constants; + group_offsets_ = problem.dispersive.group_offsets; + group_members_ = problem.dispersive.group_members; groups_ = group_offsets_.size() - 1; - keys_ = problem.dispersive_group_keys.empty() ? std::vector(groups_, 0) - : problem.dispersive_group_keys; + keys_ = problem.dispersive.group_keys.empty() ? std::vector(groups_, 0) + : problem.dispersive.group_keys; if (!symmetric_) { // One cube per distinct key, in the order the groups first name it (solprop // `dispersive_terms_layout`). cube_of_.assign(groups_, 0); for (std::size_t group = 0; group < groups_; ++group) { - const auto at = std::find(cube_keys_.begin(), cube_keys_.end(), keys_[group]); + const auto at = std::ranges::find(cube_keys_, keys_[group]); cube_of_[group] = static_cast(at - cube_keys_.begin()); if (at == cube_keys_.end()) cube_keys_.push_back(keys_[group]); @@ -728,7 +754,7 @@ DispersiveWeights::DispersiveWeights(const Problem& problem, mixtures_ = mixture_groups_.size(); static_inputs_ = RealView2("solprop_native.dispersive.static_inputs", fields_, bins_); inputs_ = RealView2("solprop_native.dispersive.inputs", fields_, bins_); - const auto terms = problem.dispersive_term_rows.size(); + const auto terms = problem.dispersive.term_rows.size(); term_multipliers_ = RealView2("solprop_native.dispersive.term_multipliers", terms, bins_); outputs_ = RealView2("solprop_native.dispersive.outputs", terms, bins_); @@ -743,17 +769,17 @@ DispersiveWeights::DispersiveWeights(const Problem& problem, term_rows_ = IndexView("solprop_native.dispersive.term_rows", terms); term_channels_ = IndexView("solprop_native.dispersive.term_channels", terms); members_ = IndexView("solprop_native.dispersive.members", channels_); - upload(kernels_, static_inputs_, problem.dispersive_inputs); + upload(kernels_, static_inputs_, problem.dispersive.inputs); graph::deep_copy(kernels_, inputs_, static_inputs_); - upload(kernels_, term_multipliers_, problem.dispersive_term_multipliers); + upload(kernels_, term_multipliers_, problem.dispersive.term_multipliers); graph::deep_copy(kernels_, outputs_, term_multipliers_); if (mixtures_ != 0) - upload(kernels_, quartics_, problem.dispersive_quartics); - upload(kernels_, input_scale_, problem.dispersive_input_scale); - upload(kernels_, output_scale_, problem.dispersive_output_scale); - upload(kernels_, term_rows_, problem.dispersive_term_rows); - upload(kernels_, term_channels_, problem.dispersive_term_channels); - upload(kernels_, members_, problem.dispersive_group_members); + upload(kernels_, quartics_, problem.dispersive.quartics); + upload(kernels_, input_scale_, problem.dispersive.input_scale); + upload(kernels_, output_scale_, problem.dispersive.output_scale); + upload(kernels_, term_rows_, problem.dispersive.term_rows); + upload(kernels_, term_channels_, problem.dispersive.term_channels); + upload(kernels_, members_, problem.dispersive.group_members); pairs_.assign(2 * groups_, 0.0); densities_.assign(n_stages_ * channels_, 0.0); last_.assign(channels_, 0.0); @@ -770,7 +796,7 @@ void DispersiveWeights::require_densities(const std::vector& densities) void DispersiveWeights::set_densities(const std::vector& densities) { require_densities(densities); - std::copy(densities.begin(), densities.end(), densities_.begin()); + std::ranges::copy(densities, densities_.begin()); } void DispersiveWeights::update(const std::size_t stage) { @@ -884,16 +910,18 @@ void DispersiveWeights::close() noexcept { DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : n_modes_(problem.n_modes), n_pol_state_(problem.n_pol_state), - n_pol_transform_(problem.n_pol_transform), + : n_modes_(problem.dims.n_modes), n_pol_state_(problem.dims.n_pol_state), + n_pol_transform_(problem.dims.n_pol_transform), components_(problem.effective_node_pol()), columns_(problem.n_columns()), - n_nodes_(problem.n_nodes), node_chunk_(problem.node_chunk), - n_chunks_((problem.n_nodes + problem.node_chunk - 1) / problem.node_chunk), - n_to_(problem.n_to), form_(static_cast(problem.dispersive_form)), + n_nodes_(problem.dims.n_nodes), node_chunk_(problem.dims.node_chunk), + n_chunks_((problem.dims.n_nodes + problem.dims.node_chunk - 1) / + problem.dims.node_chunk), + n_to_(problem.dims.n_to), + form_(static_cast(problem.dispersive.form)), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { - if (problem.general_modal || problem.separable_geometry()) + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { + if (problem.general.enabled || problem.separable_geometry()) throw std::invalid_argument( "the fused dispersive response runs on the legacy transform's node path"); const bool complex = form_ != DispersiveForm::real; @@ -902,8 +930,8 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& const auto fields = weights_.fields(); const auto products = weights_.products(); kernels_ = pool.get(CpuOperation::stage, products * columns_ * n_to_); - response_chunk_ = problem.node_chunk; - const auto tail = problem.n_nodes % response_chunk_; + response_chunk_ = problem.dims.node_chunk; + const auto tail = problem.dims.n_nodes % response_chunk_; response_ = pool.get(CpuOperation::response, response_chunk_ * components_ * n_to_, n_to_); tail_response_ = pool.get(CpuOperation::response, tail * components_ * n_to_, n_to_); @@ -914,14 +942,14 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& matrix_copy_ = pool.get(CpuOperation::copy, n_modes_ * n_nodes_ * n_pol_transform_); dense_ = pool.get(CpuOperation::dense, dense_elements); tail_dense_ = pool.get(CpuOperation::dense, tail_dense_elements); - analytic_ = problem.family == FieldFamily::field_analytic; - embedded_ = !problem.projection_coarse_matrix.empty(); + analytic_ = problem.dims.family == FieldFamily::field_analytic; + embedded_ = !problem.transform.projection_coarse_matrix.empty(); // Alone, nothing else is projected or transformed beside the response; a lone one is // also a single product, which reads the stage's own node buffer. - alone_ = !problem.nonlinear && !problem.plasma && !problem.raman && - !problem.envelope_plasma; + alone_ = !problem.kerr.nonlinear && !problem.plasma.enabled && + !problem.raman.enabled && !problem.plasma.envelope; lone_ = alone_ && symmetric && products == 1; - in_place_ = lone_ && !problem.in_stage_absorber; + in_place_ = lone_ && !problem.absorber.in_stage; borrowed_ = lone_ && !analytic_; shifted_ = analytic_ && !symmetric; @@ -1302,19 +1330,19 @@ void DispersiveResponse::close() noexcept { SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : n_to_(problem.n_to), n_x_(problem.n_x), n_p_(problem.n_p), - form_(static_cast(problem.dispersive_form)), + : n_to_(problem.dims.n_to), n_x_(problem.spatial.n_x), n_p_(problem.spatial.n_p), + form_(static_cast(problem.dispersive.form)), slab_(problem.slab_geometry()), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { if (!problem.separable_geometry()) throw std::invalid_argument("the separable dispersive response runs on the " "Cartesian free-space, line and planar routes"); // A chunk's temporal rows: a free-space chunk's nodes times their components, a // planar block's (p, y) rows; and the rows of its node buffer. components_ = problem.effective_node_pol(); - node_rows_ = problem.node_chunk * components_; - rows_ = slab_ ? (problem.node_chunk / n_x_) * n_p_ : node_rows_; + node_rows_ = problem.dims.node_chunk * components_; + rows_ = slab_ ? (problem.dims.node_chunk / n_x_) * n_p_ : node_rows_; require_size(stage_.fine_spectrum.extent(0), rows_, "chunk rows"); const bool complex = form_ != DispersiveForm::real; spectrum_bins_ = complex ? n_to_ : n_to_ / 2 + 1; @@ -1322,8 +1350,8 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo const auto fields = weights_.fields(); const auto products = weights_.products(); kernels_ = pool.get(CpuOperation::stage, products * rows_ * n_to_); - response_chunk_ = problem.node_chunk; - const auto tail = problem.n_nodes % response_chunk_; + response_chunk_ = problem.dims.node_chunk; + const auto tail = problem.dims.n_nodes % response_chunk_; active_rows_ = rows_; if constexpr (cpu_size_policy_enabled) tail_rows_ = slab_ ? (tail / n_x_) * n_p_ : tail * components_; @@ -1333,9 +1361,9 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo tail_response_ = pool.get(CpuOperation::response, tail * components_ * n_to_, n_to_); dense_ = pool.get(CpuOperation::dense, response_chunk_ * components_ * n_to_ * n_p_); tail_dense_ = pool.get(CpuOperation::dense, tail * components_ * n_to_ * n_p_); - analytic_ = problem.family == FieldFamily::field_analytic; - alone_ = !problem.nonlinear && !problem.plasma && !problem.raman && - !problem.envelope_plasma; + analytic_ = problem.dims.family == FieldFamily::field_analytic; + alone_ = !problem.kerr.nonlinear && !problem.plasma.enabled && + !problem.raman.enabled && !problem.plasma.envelope; lone_ = alone_ && symmetric && products == 1; // The in-stage absorber is a pass of its own here, which packs the chunk afresh. in_place_ = lone_; @@ -1683,14 +1711,14 @@ void SeparableDispersive::close() noexcept { GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : rows_(problem.general_node_chunk * problem.general_components), - components_(problem.general_components), n_to_(problem.n_to), - real_(static_cast(problem.dispersive_form) == + : rows_(problem.general.node_chunk * problem.general.components), + components_(problem.general.components), n_to_(problem.dims.n_to), + real_(static_cast(problem.dispersive.form) == DispersiveForm::real), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { - if (!problem.general_modal) + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { + if (!problem.general.enabled) throw std::invalid_argument( "the general dispersive response runs on the general modal route"); require_size(stage_.spectrum.extent(0), rows_, "chunk rows"); @@ -1699,10 +1727,10 @@ GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& p const auto fields = weights_.fields() * rows_; const auto products = weights_.products() * rows_; kernels_ = pool.get(CpuOperation::stage, products * n_to_); - response_chunk_ = problem.general_node_chunk; - const auto tail = problem.general_nodes % response_chunk_; + response_chunk_ = problem.general.node_chunk; + const auto tail = problem.general.nodes % response_chunk_; active_rows_ = rows_; - last_chunk_ = problem.general_nodes / response_chunk_; + last_chunk_ = problem.general.nodes / response_chunk_; if constexpr (cpu_size_policy_enabled) tail_rows_ = tail * components_; tail_kernels_ = @@ -1723,13 +1751,13 @@ GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& p if (shifted_) dc_ = ComplexView("solprop_native.dispersive.zero_frequency", rows_); node_region_ = - IndexView("solprop_native.dispersive.node_region", problem.general_nodes); - upload(kernels_, node_region_, problem.dispersive_node_region); + IndexView("solprop_native.dispersive.node_region", problem.general.nodes); + upload(kernels_, node_region_, problem.dispersive.node_region); // Which chunks have a node of any channel's region, decided once. - const auto& region = problem.dispersive_node_region; - const auto chunk = problem.general_node_chunk; - for (std::size_t first = 0; first < problem.general_nodes; first += chunk) { - const auto last = std::min(first + chunk, problem.general_nodes); + const auto& region = problem.dispersive.node_region; + const auto chunk = problem.general.node_chunk; + for (std::size_t first = 0; first < problem.general.nodes; first += chunk) { + const auto last = std::min(first + chunk, problem.general.nodes); bool any = false; for (std::size_t node = first; node < last && !any; ++node) for (std::size_t group = 0; group < weights_.groups() && !any; ++group) diff --git a/native/src/engines.cpp b/native/src/engines.cpp index 3d1c25616..bdd50ba7f 100644 --- a/native/src/engines.cpp +++ b/native/src/engines.cpp @@ -318,7 +318,7 @@ ScriptedRungeKutta::ScriptedRungeKutta(const std::vector& norms, double ScriptedRungeKutta::trial(const double z, const double dz) { if (next_ >= norms_.size()) throw std::out_of_range("the script ran out of norms"); - log_.push_back(ScriptedTrial{z, dz}); + log_.push_back(ScriptedTrial{.z = z, .dz = dz}); const double noise = norms_[next_++]; return h0_ > 0.0 ? noise * std::pow(dz / h0_, power_) : noise; } diff --git a/native/src/far_field.cpp b/native/src/far_field.cpp index 3e2666b30..4d8ed378b 100644 --- a/native/src/far_field.cpp +++ b/native/src/far_field.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/far_field.hpp" + #include "solprop_native/launch.hpp" #include @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace solprop_native { @@ -71,18 +73,19 @@ std::size_t far_field_band_count(const std::size_t n_x, const std::size_t n_y, return std::max(1, std::min(units, admitted)); } -FarFieldPass::FarFieldPass(const Problem& problem, const ExecutionSpace& execution) - : execution_(execution), n_x_(problem.n_x), n_y_(problem.n_y), - n_omega_(problem.n_omega), n_pol_(problem.n_pol_state), n_t_(problem.n_t), - dt_(problem.dt), envelope_(problem.family == FieldFamily::envelope) { - if (!problem.free_geometry() || problem.spatial_k_x.size() != n_x_ || - problem.spatial_k_y.size() != n_y_) +FarFieldPass::FarFieldPass(const Problem& problem, ExecutionSpace execution) + : execution_(std::move(execution)), n_x_(problem.spatial.n_x), + n_y_(problem.spatial.n_y), n_omega_(problem.dims.n_omega), + n_pol_(problem.dims.n_pol_state), n_t_(problem.dims.n_t), dt_(problem.grid.dt), + envelope_(problem.dims.family == FieldFamily::envelope) { + if (!problem.free_geometry() || problem.spatial.k_x.size() != n_x_ || + problem.spatial.k_y.size() != n_y_) throw std::invalid_argument( "the far-field pass needs a Cartesian or line grid and its column axes"); - for (const double value : problem.spatial_k_x) + for (const double value : problem.spatial.k_x) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_x must be finite"); - for (const double value : problem.spatial_k_y) + for (const double value : problem.spatial.k_y) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_y must be finite"); k_x_ = RealView("solprop_native.far_field.k_x", n_x_); @@ -90,13 +93,13 @@ FarFieldPass::FarFieldPass(const Problem& problem, const ExecutionSpace& executi const auto host_x = Kokkos::create_mirror_view(k_x_); const auto host_y = Kokkos::create_mirror_view(k_y_); for (std::size_t index = 0; index < n_x_; ++index) - host_x(index) = problem.spatial_k_x[index]; + host_x(index) = problem.spatial.k_x[index]; for (std::size_t index = 0; index < n_y_; ++index) - host_y(index) = problem.spatial_k_y[index]; + host_y(index) = problem.spatial.k_y[index]; Kokkos::deep_copy(execution_, k_x_, host_x); Kokkos::deep_copy(execution_, k_y_, host_y); - if (problem.spatial_far_field_slots != 0) { - const auto slots = problem.spatial_far_field_slots; + if (problem.spatial.far_field_slots != 0) { + const auto slots = problem.spatial.far_field_slots; const auto bands = far_field_band_count(n_x_, n_y_, slots, n_omega_); partials_ = RealView("solprop_native.far_field.partials", bands * slots * n_omega_); } @@ -224,7 +227,8 @@ std::vector FarFieldPass::run(const std::uint64_t flags, if (table_size != 0) Kokkos::deep_copy(execution_, tables_, host_tables_); accumulate(layout, n_bands, n_apertures, n_regions, exponent, - Sharing{shared, one_distance, circles}, field, inputs); + Sharing{.centre = shared, .distance = one_distance, .circles = circles}, + field, inputs); finish(layout, n_bands, n_apertures, n_regions, intensity_factor, packed_size); Kokkos::deep_copy(execution_, host_packed_, packed_); execution_.fence("solprop_native far-field statistics boundary"); @@ -243,7 +247,8 @@ void FarFieldPass::axis_spectrum(const bool closed, const ComplexView2& spectrum launch::parallel_for( "solprop_native.far_field.axis_spectrum", Range2(execution_, {0, 0}, {n_pol_, n}), KOKKOS_LAMBDA(const std::size_t pol, const std::size_t bin) { - double real = 0.0, imag = 0.0; + double real = 0.0; + double imag = 0.0; for (std::size_t band = 0; band < n_bands; ++band) { real += partials((band * slots + first + 2 * pol) * n + bin); imag += partials((band * slots + first + 2 * pol + 1) * n + bin); @@ -408,12 +413,14 @@ void FarFieldPass::accumulate(const FarFieldLayout& layout, const std::size_t n_ transmission = far_field_shaped_weight( far_field_offset(reach, kx, centre_x), far_field_offset(reach, ky, centre_y), - FarFieldOutline{tables(constants + far_field_half_width), - tables(constants + far_field_half_height), - tables(constants + far_field_stretch_x), - tables(constants + far_field_stretch_y), - tables(constants + far_field_inverse_width), - tables(constants + far_field_edge)}, + FarFieldOutline{ + .half_width = tables(constants + far_field_half_width), + .half_height = tables(constants + far_field_half_height), + .stretch_x = tables(constants + far_field_stretch_x), + .stretch_y = tables(constants + far_field_stretch_y), + .inverse_width = + tables(constants + far_field_inverse_width), + .edge = tables(constants + far_field_edge)}, shape, code); } if (transmission == 0.0) @@ -462,7 +469,7 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band const double density = envelope ? dt_ * dt_ / (2.0 * std::numbers::pi) : 2.0 * (dt_ * dt_) / std::numbers::pi; const double energy = dt_ / static_cast(n_t_); - const std::size_t leading = static_cast(layout.beam >= 0); + const auto leading = static_cast(layout.beam >= 0); const std::size_t absorbed = layout.absorbed >= 0 ? far_field_absorbed_channels : 0; const std::size_t spectral = (packed_size - absorbed) / n; const std::size_t regions = layout.regions >= 0 ? n_regions : 0; @@ -482,6 +489,8 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band std::size_t rest = row; long slot = -1; bool amplitude = false; + // Each test takes the previous range off the row before comparing. + // NOLINTBEGIN(bugprone-assignment-in-if-condition): a packed-row cascade if (rest < leading) { slot = layout.beam; } else if ((rest -= leading) < regions) { @@ -497,6 +506,7 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band slot = layout.axis; amplitude = true; } + // NOLINTEND(bugprone-assignment-in-if-condition) const auto first = static_cast(slot); double value = 0.0; if (amplitude) { diff --git a/native/src/general.cpp b/native/src/general.cpp index c8b3954d4..8e316fd0b 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -3,9 +3,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/adams_phi.hpp" +#include "solprop_native/build.hpp" #include "solprop_native/core.hpp" #include "solprop_native/coupled_flow.hpp" #include "solprop_native/crossing.hpp" +#include "solprop_native/dispatch.hpp" #include "solprop_native/dispersive.hpp" #include "solprop_native/envelope_plasma.hpp" #include "solprop_native/far_field.hpp" @@ -15,16 +17,17 @@ #include "solprop_native/gemm.hpp" #include "solprop_native/general_modal.hpp" #include "solprop_native/plasma.hpp" - #include "solprop_native/raman_fft.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/slab.hpp" #include "solprop_native/vmath.hpp" #include "solprop_native/z_table.hpp" + #include #include #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) #include -#if defined(__SSE2__) +#ifdef __SSE2__ #include #define SOLPROP_NATIVE_STREAMING_STORES 1 #endif @@ -39,19 +42,22 @@ #include #include #include +#include #include #include #include #include +#include #include #include -#if defined(__APPLE__) +#ifdef __APPLE__ #include #elif defined(__GLIBC__) #include #endif namespace solprop_native { +namespace { /** The three squared norms an accepted step reports, as one reduction value. */ struct NormSums { double old_sum = 0.0; @@ -64,6 +70,7 @@ struct NormSums { return *this; } }; +} // namespace } // namespace solprop_native template <> struct Kokkos::reduction_identity { @@ -71,6 +78,7 @@ template <> struct Kokkos::reduction_identity { }; namespace solprop_native { +namespace { /** A plane's scaled fluence, one reduction value: its sum and its index moments. */ struct FluenceMoments { double total = 0.0; @@ -83,6 +91,7 @@ struct FluenceMoments { return *this; } }; +} // namespace } // namespace solprop_native template <> struct Kokkos::reduction_identity { @@ -178,12 +187,13 @@ void apply_generator(const ExecutionSpace& space, const std::size_t n, #endif } -template KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&...) {} +template +KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&... /*unused*/) {} // Net process allocator growth includes opaque vendor FFT plan/workspace // allocations. It is measured separately from Kokkos Views and is not a peak bound. std::size_t heap_bytes() { -#if defined(__APPLE__) +#ifdef __APPLE__ malloc_statistics_t statistics{}; malloc_zone_statistics(nullptr, &statistics); return statistics.size_in_use; @@ -208,7 +218,7 @@ std::size_t heap_bytes() { #endif } std::size_t device_free_bytes() { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA std::size_t available{}, total{}; if (cudaMemGetInfo(&available, &total) != cudaSuccess) throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); @@ -284,7 +294,7 @@ using StripPlan = KokkosFFT::Plan struct EnvelopeStream { ComplexView2 node, projected, coarse, stacked, spectrum, coarse_spectrum; std::unique_ptr forward, coarse_forward; - std::size_t bytes() const { + [[nodiscard]] std::size_t bytes() const { return sizeof(Complex) * (node.size() + projected.size() + coarse.size() + stacked.size() + spectrum.size() + coarse_spectrum.size()); @@ -296,7 +306,7 @@ struct FieldCurrentStream { RealView2 node, projected, coarse, stacked; ComplexView2 spectrum, coarse_spectrum; std::unique_ptr forward, coarse_forward; - std::size_t bytes() const { + [[nodiscard]] std::size_t bytes() const { return sizeof(double) * (node.size() + projected.size() + coarse.size() + stacked.size()) + sizeof(Complex) * (spectrum.size() + coarse_spectrum.size()); @@ -316,6 +326,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, label, Kokkos::RangePolicy(execution, 0, rows), KOKKOS_LAMBDA(const std::size_t row) { for (std::size_t inner = 0; inner < width; ++inner) + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide element(row, inner); }); #endif @@ -337,7 +348,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, // builds compile out the host traversals that use these, hence `maybe_unused`. [[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_store(Complex& target, const Complex value) { -#if defined(SOLPROP_NATIVE_STREAMING_STORES) +#ifdef SOLPROP_NATIVE_STREAMING_STORES _mm_stream_pd(reinterpret_cast(&target), _mm_set_pd(value.imag(), value.real())); #else @@ -346,7 +357,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, } [[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_fence() { -#if defined(SOLPROP_NATIVE_STREAMING_STORES) +#ifdef SOLPROP_NATIVE_STREAMING_STORES _mm_sfence(); #endif } @@ -357,11 +368,12 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, struct CrossingTile { std::size_t first_row, last_row, first_bin, last_bin; - std::size_t size() const noexcept { + [[nodiscard]] std::size_t size() const noexcept { return (last_row - first_row) * (last_bin - first_bin); } // An element's place in a tile buffer, rows outermost. - std::size_t at(const std::size_t row, const std::size_t bin) const noexcept { + [[nodiscard]] std::size_t at(const std::size_t row, + const std::size_t bin) const noexcept { return (row - first_row) * (last_bin - first_bin) + (bin - first_bin); } // Every element in the traversal order `crossing_for` names. @@ -393,8 +405,10 @@ void crossing_tiles(const char* label, const ExecutionSpace& execution, const std::size_t first_row = (index / bin_tiles) * tile_rows; const std::size_t first_bin = (index % bin_tiles) * tile_bins; visit(CrossingTile{ - first_row, rows - first_row < tile_rows ? rows : first_row + tile_rows, - first_bin, width - first_bin < tile_bins ? width : first_bin + tile_bins}); + .first_row = first_row, + .last_row = rows - first_row < tile_rows ? rows : first_row + tile_rows, + .first_bin = first_bin, + .last_bin = width - first_bin < tile_bins ? width : first_bin + tile_bins}); // Each thread orders its own streaming stores before the region ends. if constexpr (Streaming) stream_fence(); @@ -615,7 +629,7 @@ inline void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, fftw_plan_with_nthreads(space.concurrency()); auto* in = reinterpret_cast(input); auto* out = reinterpret_cast(output); - const fftw_plan seeded = fftw_plan_many_dft( + auto* const seeded = fftw_plan_many_dft( static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, input_distance, out, nullptr, 1, output_distance, sign, *chosen); // The policy has always tolerated a plane it cannot seed; a profile records it. @@ -814,7 +828,8 @@ struct SpatialCoefficients { const Complex prefactor = propagating ? numerator(omega) / root.real() : clamped(omega); #endif - return {linear - 0.5 * spectral_rate(omega), active ? prefactor : Complex{}}; + return {.linear = linear - 0.5 * spectral_rate(omega), + .prefactor = active ? prefactor : Complex{}}; } }; @@ -961,6 +976,7 @@ ionisation_log_rate(const double log_field, const View& coefficients, long raw_index = static_cast((log_field - log_field_min) / d_log_field); if (raw_index < 0) raw_index = 0; + // NOLINTNEXTLINE(modernize-use-integer-sign-comparison): device code; std::cmp_* is host-only if (raw_index >= static_cast(intervals)) raw_index = static_cast(intervals - 1); const auto index = static_cast(raw_index); @@ -1081,288 +1097,308 @@ void require_general_problem(const Problem& problem) { require_general_modal(problem); require_coupled_flow(problem); require_dispersive(problem); - if (problem.n_omega == 0 || problem.n_t == 0 || problem.n_to == 0 || - problem.n_modes == 0 || problem.n_pol_state == 0 || problem.n_nodes == 0 || - problem.n_pol_transform == 0) + if (problem.dims.n_omega == 0 || problem.dims.n_t == 0 || problem.dims.n_to == 0 || + problem.dims.n_modes == 0 || problem.dims.n_pol_state == 0 || + problem.dims.n_nodes == 0 || problem.dims.n_pol_transform == 0) throw std::invalid_argument("all transform and FFT extents must be positive"); - const auto effective_node_pol = checked_product( - problem.n_pol_transform, problem.n_pol_state, "effective node polarisation"); + const auto effective_node_pol = + checked_product(problem.dims.n_pol_transform, problem.dims.n_pol_state, + "effective node polarisation"); if (effective_node_pol > 2) throw std::invalid_argument("at most two effective node components are supported"); - if (problem.node_chunk == 0 || problem.node_chunk > problem.n_nodes) + if (problem.dims.node_chunk == 0 || problem.dims.node_chunk > problem.dims.n_nodes) throw std::invalid_argument("node_chunk must lie in [1, n_nodes]"); - if (problem.n_stages == 0) + if (problem.stepper.n_stages == 0) throw std::invalid_argument("the tableau must contain a stage"); const auto columns = - checked_product(problem.n_modes, problem.n_pol_state, "field columns"); + checked_product(problem.dims.n_modes, problem.dims.n_pol_state, "field columns"); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) constexpr auto policy_limit = static_cast(std::numeric_limits::max()); - if (problem.n_to > policy_limit || problem.n_t > policy_limit || - problem.n_omega > policy_limit || columns > policy_limit || - problem.node_chunk > policy_limit) + if (problem.dims.n_to > policy_limit || problem.dims.n_t > policy_limit || + problem.dims.n_omega > policy_limit || columns > policy_limit || + problem.dims.node_chunk > policy_limit) throw std::length_error("accelerator policy extent exceeds signed 32-bit range"); #endif - const auto rows = - checked_product(problem.n_nodes, problem.n_pol_transform, "transform rows"); - const auto field = checked_product(columns, problem.n_omega, "field"); - const auto modal_time = checked_product(columns, problem.n_to, "modal time"); + const auto rows = checked_product(problem.dims.n_nodes, problem.dims.n_pol_transform, + "transform rows"); + const auto field = checked_product(columns, problem.dims.n_omega, "field"); + const auto modal_time = checked_product(columns, problem.dims.n_to, "modal time"); const auto chunk_rows = - checked_product(problem.node_chunk, effective_node_pol, "node chunk rows"); - const auto node_time = checked_product(chunk_rows, problem.n_to, "node time"); + checked_product(problem.dims.node_chunk, effective_node_pol, "node chunk rows"); + const auto node_time = checked_product(chunk_rows, problem.dims.n_to, "node time"); // The hybrid planar kind carries its x half alone, (n_x, n_p) and its transpose. const auto transform = problem.free_geometry() ? 0 : problem.slab_geometry() - ? checked_product(problem.n_x, problem.n_p, "slab transform matrix") - : checked_product(rows, problem.n_modes, "transform matrix"); - const auto stage_field = checked_product(problem.n_stages, field, "stage field"); + ? checked_product(problem.spatial.n_x, problem.spatial.n_p, + "slab transform matrix") + : checked_product(rows, problem.dims.n_modes, "transform matrix"); + const auto stage_field = + checked_product(problem.stepper.n_stages, field, "stage field"); require_allocation(field, sizeof(Complex), "field bytes"); require_allocation(modal_time, sizeof(Complex), "modal-time bytes"); require_allocation(node_time, sizeof(Complex), "node-time bytes"); require_allocation(transform, sizeof(double), "transform-matrix bytes"); - if (!problem.projection_coarse_matrix.empty() && - problem.family != FieldFamily::envelope) { + if (!problem.transform.projection_coarse_matrix.empty() && + problem.dims.family != FieldFamily::envelope) { const auto stacked_rows = - checked_product(2, problem.n_modes, "stacked projection rows"); + checked_product(2, problem.dims.n_modes, "stacked projection rows"); require_allocation( - checked_product(stacked_rows, problem.n_to, "stacked projection"), + checked_product(stacked_rows, problem.dims.n_to, "stacked projection"), sizeof(double), "stacked-projection bytes"); } require_allocation(stage_field, sizeof(Complex), "stage-field bytes"); - if (problem.plasma && !problem.envelope_plasma) { + if (problem.plasma.enabled && !problem.plasma.envelope) { require_allocation( - checked_product(chunk_rows, problem.n_omega, "held nodal spectrum"), + checked_product(chunk_rows, problem.dims.n_omega, "held nodal spectrum"), sizeof(Complex), "held nodal spectrum bytes"); require_allocation(checked_product(2, modal_time, "held stacked projection"), sizeof(double), "held stacked projection bytes"); - require_allocation( - checked_product(problem.n_modes, - checked_product(problem.node_chunk, problem.n_pol_transform, - "held projection rows"), - "held projection"), - sizeof(double), "held projection bytes"); - } - require_extent(problem.crop_index.size(), problem.n_omega, "crop_index"); - require_extent(problem.active_mask.size(), problem.n_omega, "active_mask"); - require_extent(problem.norm_mask.size(), problem.n_omega, "norm_mask"); - require_extent(problem.omega_window.size(), problem.n_omega, "omega_window"); - if (!problem.omega.empty()) - require_extent(problem.omega.size(), problem.n_omega, "omega"); - if (!problem.spatial_spectral_weights.empty()) { - require_extent(problem.spatial_spectral_weights.size(), problem.n_modes, + require_allocation(checked_product(problem.dims.n_modes, + checked_product(problem.dims.node_chunk, + problem.dims.n_pol_transform, + "held projection rows"), + "held projection"), + sizeof(double), "held projection bytes"); + } + require_extent(problem.grid.crop_index.size(), problem.dims.n_omega, "crop_index"); + require_extent(problem.grid.active_mask.size(), problem.dims.n_omega, "active_mask"); + require_extent(problem.grid.norm_mask.size(), problem.dims.n_omega, "norm_mask"); + require_extent(problem.grid.omega_window.size(), problem.dims.n_omega, + "omega_window"); + if (!problem.grid.omega.empty()) + require_extent(problem.grid.omega.size(), problem.dims.n_omega, "omega"); + if (!problem.spatial.spectral_weights.empty()) { + require_extent(problem.spatial.spectral_weights.size(), problem.dims.n_modes, "spatial_spectral_weights"); - require_extent(problem.spatial_on_axis.size(), problem.n_modes, "spatial_on_axis"); - if ((!problem.slab_geometry() && problem.n_nodes != problem.n_modes) || - problem.n_pol_transform != 1) + require_extent(problem.spatial.on_axis.size(), problem.dims.n_modes, + "spatial_on_axis"); + if ((!problem.slab_geometry() && problem.dims.n_nodes != problem.dims.n_modes) || + problem.dims.n_pol_transform != 1) throw std::invalid_argument( "radial statistics require a scalar square transform"); - for (const double weight : problem.spatial_spectral_weights) + for (const double weight : problem.spatial.spectral_weights) if (!std::isfinite(weight) || weight <= 0.0) throw std::invalid_argument( "spatial spectral weights must be finite and positive"); - } else if (!problem.spatial_on_axis.empty()) { + } else if (!problem.spatial.on_axis.empty()) { throw std::invalid_argument("spatial on-axis weights require the spectral measure"); } if (problem.slab_geometry()) { // solprop's hybrid planar kind: columns (p, k_y), nodes (x, y), a scalar state, // whole y rows per node chunk and no embedded coarse rule (solprop's reference // scores no quadrature alarm on this kind). - if (problem.n_x < 1 || problem.n_y < 1 || - checked_product(problem.n_p, problem.n_y, "planar columns") != - problem.n_modes || - checked_product(problem.n_x, problem.n_y, "planar nodes") != problem.n_nodes || - problem.n_pol_transform != 1 || problem.n_pol_state != 1 || - problem.node_chunk % problem.n_x != 0 || problem.n_phi_terms != 1 || - problem.n_prefactor_terms != 1 || !problem.projection_coarse_matrix.empty()) + if (problem.spatial.n_x < 1 || problem.spatial.n_y < 1 || + checked_product(problem.spatial.n_p, problem.spatial.n_y, "planar columns") != + problem.dims.n_modes || + checked_product(problem.spatial.n_x, problem.spatial.n_y, "planar nodes") != + problem.dims.n_nodes || + problem.dims.n_pol_transform != 1 || problem.dims.n_pol_state != 1 || + problem.dims.node_chunk % problem.spatial.n_x != 0 || + problem.linear.n_phi_terms != 1 || problem.linear.n_prefactor_terms != 1 || + !problem.transform.projection_coarse_matrix.empty()) throw std::invalid_argument( "the hybrid planar kind requires matching axes, a scalar state, whole y " "rows per node chunk and scalar separable data"); - require_extent(problem.spatial_kappa_sq.size(), - checked_product(problem.n_p, problem.n_omega, "planar wall terms"), - "spatial_kappa_sq"); - for (const auto value : problem.spatial_kappa_sq) + require_extent( + problem.spatial.kappa_sq.size(), + checked_product(problem.spatial.n_p, problem.dims.n_omega, "planar wall terms"), + "spatial_kappa_sq"); + for (const auto value : problem.spatial.kappa_sq) if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) throw std::invalid_argument("spatial_kappa_sq must be finite"); - } else if (!problem.spatial_kappa_sq.empty()) { + } else if (!problem.spatial.kappa_sq.empty()) { throw std::invalid_argument("spatial_kappa_sq belongs to the hybrid planar kind"); } // n_y == 1 is solprop's line uniform in y (grid.LineGrid): one y row carrying // k_y = 0, transformed by the same two-axis plan with a unit extent. if (problem.free_geometry() && - (problem.n_x < 2 || problem.n_y < 1 || - checked_product(problem.n_x, problem.n_y, "Cartesian shape") != - problem.n_modes || - problem.n_nodes != problem.n_modes || problem.n_pol_transform != 1 || - problem.n_phi_terms != 1 || problem.n_prefactor_terms != 1)) + (problem.spatial.n_x < 2 || problem.spatial.n_y < 1 || + checked_product(problem.spatial.n_x, problem.spatial.n_y, "Cartesian shape") != + problem.dims.n_modes || + problem.dims.n_nodes != problem.dims.n_modes || + problem.dims.n_pol_transform != 1 || problem.linear.n_phi_terms != 1 || + problem.linear.n_prefactor_terms != 1)) throw std::invalid_argument( "Cartesian geometry requires matching axes and scalar separable data"); if (problem.separable_geometry()) { - if (!std::isfinite(problem.spatial_evanescent_rate) || - problem.spatial_evanescent_rate <= 0) + if (!std::isfinite(problem.spatial.evanescent_rate) || + problem.spatial.evanescent_rate <= 0) throw std::invalid_argument( "Cartesian evanescent rate must be finite and positive"); - require_extent(problem.spatial_k_squared.size(), problem.n_omega, + require_extent(problem.spatial.k_squared.size(), problem.dims.n_omega, "spatial_k_squared"); - require_extent(problem.spatial_k_squared_low.size(), problem.n_omega, + require_extent(problem.spatial.k_squared_low.size(), problem.dims.n_omega, "spatial_k_squared_low"); - require_extent(problem.spatial_numerator.size(), problem.n_omega, + require_extent(problem.spatial.numerator.size(), problem.dims.n_omega, "spatial_numerator"); - require_extent(problem.spatial_clamped.size(), problem.n_omega, "spatial_clamped"); - require_extent(problem.spatial_frame.size(), problem.n_omega, "spatial_frame"); - require_extent(problem.spatial_spectral_rate.size(), problem.n_omega, + require_extent(problem.spatial.clamped.size(), problem.dims.n_omega, + "spatial_clamped"); + require_extent(problem.spatial.frame.size(), problem.dims.n_omega, "spatial_frame"); + require_extent(problem.spatial.spectral_rate.size(), problem.dims.n_omega, "spatial_spectral_rate"); - require_extent(problem.spatial_user_rate.size(), problem.n_omega, + require_extent(problem.spatial.user_rate.size(), problem.dims.n_omega, "spatial_user_rate"); - require_extent(problem.spatial_kperp_sq.size(), problem.n_modes, + require_extent(problem.spatial.kperp_sq.size(), problem.dims.n_modes, "spatial_kperp_sq"); - require_extent(problem.spatial_kperp_sq_low.size(), problem.n_modes, + require_extent(problem.spatial.kperp_sq_low.size(), problem.dims.n_modes, "spatial_kperp_sq_low"); - require_extent(problem.spatial_absorber_rate.size(), problem.n_modes, + require_extent(problem.spatial.absorber_rate.size(), problem.dims.n_modes, "spatial_absorber_rate"); - require_extent(problem.spatial_kperp_rate.size(), problem.n_modes, + require_extent(problem.spatial.kperp_rate.size(), problem.dims.n_modes, "spatial_kperp_rate"); - for (const auto value : problem.spatial_k_squared) + for (const auto value : problem.spatial.k_squared) if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) throw std::invalid_argument("spatial_k_squared must be finite"); - for (const auto value : problem.spatial_k_squared_low) + for (const auto value : problem.spatial.k_squared_low) if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) throw std::invalid_argument("spatial_k_squared_low must be finite"); - for (const auto value : problem.spatial_numerator) + for (const auto value : problem.spatial.numerator) if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) throw std::invalid_argument("spatial_numerator must be finite"); - for (const auto value : problem.spatial_clamped) + for (const auto value : problem.spatial.clamped) if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) throw std::invalid_argument("spatial_clamped must be finite"); - for (const auto value : problem.spatial_frame) + for (const auto value : problem.spatial.frame) if (!std::isfinite(value)) throw std::invalid_argument("spatial_frame is invalid"); - for (const auto value : problem.spatial_spectral_rate) + for (const auto value : problem.spatial.spectral_rate) if (!std::isfinite(value) || value < 0) throw std::invalid_argument("spatial_spectral_rate is invalid"); - for (std::size_t bin = 0; bin < problem.n_omega; ++bin) - if (const double value = problem.spatial_user_rate[bin]; + for (std::size_t bin = 0; bin < problem.dims.n_omega; ++bin) + if (const double value = problem.spatial.user_rate[bin]; !std::isfinite(value) || value < 0 || - value > problem.spatial_spectral_rate[bin]) + value > problem.spatial.spectral_rate[bin]) throw std::invalid_argument("spatial_user_rate is invalid"); - for (const auto value : problem.spatial_kperp_sq) + for (const auto value : problem.spatial.kperp_sq) if (!std::isfinite(value) || value < 0) throw std::invalid_argument("spatial_kperp_sq is invalid"); - for (const auto value : problem.spatial_kperp_sq_low) + for (const auto value : problem.spatial.kperp_sq_low) if (!std::isfinite(value)) throw std::invalid_argument("spatial_kperp_sq_low must be finite"); - for (const auto value : problem.spatial_absorber_rate) + for (const auto value : problem.spatial.absorber_rate) if (!std::isfinite(value) || value < 0) throw std::invalid_argument("spatial_absorber_rate is invalid"); - for (const auto value : problem.spatial_kperp_rate) + for (const auto value : problem.spatial.kperp_rate) if (!std::isfinite(value) || value < 0) throw std::invalid_argument("spatial_kperp_rate is invalid"); if (problem.free_geometry()) { // solprop SEP 23: the stored transverse axes the far-field statistics read. - require_extent(problem.spatial_k_x.size(), problem.n_x, "spatial_k_x"); - require_extent(problem.spatial_k_y.size(), problem.n_y, "spatial_k_y"); - for (const auto value : problem.spatial_k_x) + require_extent(problem.spatial.k_x.size(), problem.spatial.n_x, "spatial_k_x"); + require_extent(problem.spatial.k_y.size(), problem.spatial.n_y, "spatial_k_y"); + for (const auto value : problem.spatial.k_x) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_x must be finite"); - for (const auto value : problem.spatial_k_y) + for (const auto value : problem.spatial.k_y) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_y must be finite"); } - } else if (problem.n_y != 0 || problem.n_p != 0) { + } else if (problem.spatial.n_y != 0 || problem.spatial.n_p != 0) { throw std::invalid_argument("Cartesian axes must be supplied together"); } if (!problem.free_geometry() && - (!problem.spatial_k_x.empty() || !problem.spatial_k_y.empty() || - problem.spatial_far_field_slots != 0)) + (!problem.spatial.k_x.empty() || !problem.spatial.k_y.empty() || + problem.spatial.far_field_slots != 0)) throw std::invalid_argument("the far-field statistics belong to free space"); - require_extent(problem.time_window.size(), problem.n_to, "time_window"); - if (problem.plasma && !problem.envelope_plasma) { - require_extent(problem.time_window_derivative.size(), problem.n_to, + require_extent(problem.grid.time_window.size(), problem.dims.n_to, "time_window"); + if (problem.plasma.enabled && !problem.plasma.envelope) { + require_extent(problem.grid.time_window_derivative.size(), problem.dims.n_to, "time_window_derivative"); - require_extent(problem.omega.size(), problem.n_omega, "held current omega"); - for (const auto value : problem.time_window_derivative) + require_extent(problem.grid.omega.size(), problem.dims.n_omega, + "held current omega"); + for (const auto value : problem.grid.time_window_derivative) if (!std::isfinite(value)) throw std::invalid_argument("time_window_derivative is not finite"); - for (std::size_t index = 0; index < problem.n_omega; ++index) - if (problem.active_mask[index] && - (!(problem.omega[index] > 0.0) || !std::isfinite(problem.omega[index]))) + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + if (problem.grid.active_mask[index] != 0 && + (!(problem.grid.omega[index] > 0.0) || + !std::isfinite(problem.grid.omega[index]))) throw std::invalid_argument( "held plasma current requires positive active omega"); } - require_extent(problem.carrier.size(), problem.n_to, "carrier"); - require_extent(problem.synthesis_matrix.size(), transform, "synthesis_matrix"); - require_extent(problem.projection_matrix.size(), transform, "projection_matrix"); - if (!problem.projection_coarse_matrix.empty()) - require_extent(problem.projection_coarse_matrix.size(), transform, + require_extent(problem.grid.carrier.size(), problem.dims.n_to, "carrier"); + require_extent(problem.transform.synthesis_matrix.size(), transform, + "synthesis_matrix"); + require_extent(problem.transform.projection_matrix.size(), transform, + "projection_matrix"); + if (!problem.transform.projection_coarse_matrix.empty()) + require_extent(problem.transform.projection_coarse_matrix.size(), transform, "projection_coarse_matrix"); // A z table's terms are slots its evaluation fills, so nothing crosses for them. - if ((problem.phi_table || problem.prefactor_table) && problem.separable_geometry()) + if ((problem.linear.phi_table || problem.linear.prefactor_table) && + problem.separable_geometry()) throw std::invalid_argument("a separable operator takes no z table"); - require_extent(problem.phi_terms.size(), - problem.separable_geometry() || problem.phi_table + require_extent(problem.linear.phi_terms.size(), + problem.separable_geometry() || problem.linear.phi_table ? 0 - : checked_product(problem.n_phi_terms, field, "phi_terms"), + : checked_product(problem.linear.n_phi_terms, field, "phi_terms"), "phi_terms"); require_extent( - problem.prefactor_terms.size(), - problem.separable_geometry() || problem.prefactor_table + problem.linear.prefactor_terms.size(), + problem.separable_geometry() || problem.linear.prefactor_table ? 0 - : checked_product(problem.n_prefactor_terms, field, "prefactor_terms"), + : checked_product(problem.linear.n_prefactor_terms, field, "prefactor_terms"), "prefactor_terms"); - require_extent(problem.tableau_matrix.size(), - checked_product(problem.n_stages, problem.n_stages, "tableau"), - "tableau_matrix"); - require_extent(problem.tableau_nodes.size(), problem.n_stages, "tableau_nodes"); - require_extent(problem.solution_weights.size(), problem.n_stages, "solution_weights"); - require_extent(problem.error_weights.size(), problem.n_stages, "error_weights"); - if (problem.in_stage_absorber) - require_extent(problem.absorber_fine.size(), problem.n_to, "absorber_fine"); - if (!problem.spatial_absorber_matrix.empty()) { + require_extent( + problem.stepper.tableau_matrix.size(), + checked_product(problem.stepper.n_stages, problem.stepper.n_stages, "tableau"), + "tableau_matrix"); + require_extent(problem.stepper.tableau_nodes.size(), problem.stepper.n_stages, + "tableau_nodes"); + require_extent(problem.stepper.solution_weights.size(), problem.stepper.n_stages, + "solution_weights"); + require_extent(problem.stepper.error_weights.size(), problem.stepper.n_stages, + "error_weights"); + if (problem.absorber.in_stage) + require_extent(problem.absorber.fine.size(), problem.dims.n_to, "absorber_fine"); + if (!problem.spatial.absorber_matrix.empty()) { require_extent( - problem.spatial_absorber_matrix.size(), - checked_product(problem.n_modes, problem.n_modes, "spatial absorber"), + problem.spatial.absorber_matrix.size(), + checked_product(problem.dims.n_modes, problem.dims.n_modes, "spatial absorber"), "spatial_absorber_matrix"); - if (!problem.in_stage_absorber || problem.n_pol_transform != 1 || - problem.n_nodes != problem.n_modes) + if (!problem.absorber.in_stage || problem.dims.n_pol_transform != 1 || + problem.dims.n_nodes != problem.dims.n_modes) throw std::invalid_argument("spatial absorber requires scalar radial geometry"); } - if (problem.split_step_absorber) - require_extent(problem.absorber_coarse.size(), problem.n_t, "absorber_coarse"); - const std::size_t fine_size = - problem.family == FieldFamily::field_real ? problem.n_to / 2 + 1 : problem.n_to; - if (problem.envelope_plasma && problem.family != FieldFamily::envelope) + if (problem.absorber.split_step) + require_extent(problem.absorber.coarse.size(), problem.dims.n_t, "absorber_coarse"); + const std::size_t fine_size = problem.dims.family == FieldFamily::field_real + ? problem.dims.n_to / 2 + 1 + : problem.dims.n_to; + if (problem.plasma.envelope && problem.dims.family != FieldFamily::envelope) throw std::invalid_argument("envelope plasma requires an envelope field family"); std::vector seen(fine_size, false); - for (const auto index : problem.crop_index) { + for (const auto index : problem.grid.crop_index) { if (index >= fine_size) throw std::invalid_argument("crop_index lies outside the fine spectrum"); if (seen[index]) throw std::invalid_argument("crop_index contains a duplicate fine-spectrum bin"); seen[index] = true; } - if (problem.plasma && problem.n_channels == 0) + if (problem.plasma.enabled && problem.plasma.n_channels == 0) throw std::invalid_argument("plasma requires at least one channel"); - if (problem.n_channels > 1 || problem.envelope_plasma) { - if (!problem.plasma) + if (problem.plasma.n_channels > 1 || problem.plasma.envelope) { + if (!problem.plasma.enabled) throw std::invalid_argument("channels require plasma"); - require_extent(problem.channel_parameters.size(), - checked_product(problem.n_channels, 9, "channel parameters"), + require_extent(problem.plasma.channel_parameters.size(), + checked_product(problem.plasma.n_channels, 9, "channel parameters"), "channel_parameters"); - if (!std::all_of(problem.channel_parameters.begin(), - problem.channel_parameters.end(), - [](double value) { return std::isfinite(value); }) || - !std::all_of(problem.channel_coefficients.begin(), - problem.channel_coefficients.end(), - [](double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(problem.plasma.channel_parameters, + [](double value) { return std::isfinite(value); }) || + !std::ranges::all_of(problem.plasma.channel_coefficients, + [](double value) { return std::isfinite(value); })) throw std::invalid_argument("channel parameters and coefficients must be finite"); std::size_t expected_offset = 0; - for (std::size_t channel = 0; channel < problem.n_channels; ++channel) { - const double* row = problem.channel_parameters.data() + 9 * channel; + // Each interval holds four coefficients. + const std::size_t total_intervals = problem.plasma.channel_coefficients.size() / 4; + for (std::size_t channel = 0; channel < problem.plasma.n_channels; ++channel) { + const double* row = problem.plasma.channel_parameters.data() + 9 * channel; if (!(row[1] > 0.0 && row[2] > 0.0 && row[3] > row[2] && row[4] >= 0.0 && row[5] >= 1.0 && row[5] == std::floor(row[5]) && row[6] == static_cast(expected_offset) && row[7] > 0.0 && row[8] > 0.0) || - row[5] > static_cast(problem.channel_coefficients.size() / 4)) + row[5] > static_cast(total_intervals)) throw std::invalid_argument( "channel constants and coefficient ranges must be positive and consistent"); expected_offset = checked_sum( @@ -1370,79 +1406,84 @@ void require_general_problem(const Problem& problem) { checked_product(4, static_cast(row[5]), "channel table"), "channel table offsets"); } - require_extent(problem.channel_coefficients.size(), expected_offset, + require_extent(problem.plasma.channel_coefficients.size(), expected_offset, "channel_coefficients"); } - if (problem.raman && problem.n_raman_components == 0) + if (problem.raman.enabled && problem.raman.n_components == 0) throw std::invalid_argument("Raman requires at least one component"); - if (problem.n_raman_components > 1) { - if (!problem.raman) + if (problem.raman.n_components > 1) { + if (!problem.raman.enabled) throw std::invalid_argument("Raman components require Raman"); - require_extent(problem.raman_row_component.size(), problem.n_raman_terms, + require_extent(problem.raman.row_component.size(), problem.raman.n_terms, "raman_row_component"); - require_extent(problem.raman_component_fraction.size(), problem.n_raman_components, + require_extent(problem.raman.component_fraction.size(), problem.raman.n_components, "raman_component_fraction"); - for (double value : problem.raman_row_component) + for (double value : problem.raman.row_component) if (!std::isfinite(value) || value < 0.0 || - value >= static_cast(problem.n_raman_components) || + value >= static_cast(problem.raman.n_components) || value != std::floor(value)) throw std::invalid_argument("Raman row component index is invalid"); - for (double value : problem.raman_component_fraction) + for (double value : problem.raman.component_fraction) if (!std::isfinite(value) || value <= 0.0) throw std::invalid_argument( "Raman component fractions must be finite and positive"); } - if (problem.plasma) { - if (!(problem.dt_fine > 0.0) || !(problem.d_log_field > 0.0) || - !(problem.field_min > 0.0) || !(problem.field_max > problem.field_min) || - !(problem.charge_mass_ratio > 0.0) || !(problem.ionisation_potential > 0.0) || - problem.rate_intervals == 0) + if (problem.plasma.enabled) { + if (!(problem.grid.dt_fine > 0.0) || !(problem.plasma.d_log_field > 0.0) || + !(problem.plasma.field_min > 0.0) || + !(problem.plasma.field_max > problem.plasma.field_min) || + !(problem.plasma.charge_mass_ratio > 0.0) || + !(problem.plasma.ionisation_potential > 0.0) || + problem.plasma.rate_intervals == 0) throw std::invalid_argument("plasma constants must be positive and consistent"); - require_extent(problem.rate_coefficients.size(), - checked_product(4, problem.rate_intervals, "rate coefficients"), - "rate_coefficients"); + require_extent( + problem.plasma.rate_coefficients.size(), + checked_product(4, problem.plasma.rate_intervals, "rate coefficients"), + "rate_coefficients"); } // solprop SEP 18's running-integral rule. Translation certified the taps; only their // extent, finiteness, the trapezoid's identity tap and the route are checked here. - if (problem.plasma_taps.empty() || problem.plasma_taps.size() > plasma_max_taps || - !std::all_of(problem.plasma_taps.begin(), problem.plasma_taps.end(), - [](double value) { return std::isfinite(value); })) + if (problem.plasma.taps.empty() || problem.plasma.taps.size() > plasma_max_taps || + !std::ranges::all_of(problem.plasma.taps, + [](double value) { return std::isfinite(value); })) throw std::invalid_argument("plasma_taps must hold one to nine finite taps"); - if (!problem.plasma_band && - (problem.plasma_taps.size() != 1 || problem.plasma_taps[0] != 1.0)) + if (!problem.plasma.band && + (problem.plasma.taps.size() != 1 || problem.plasma.taps[0] != 1.0)) throw std::invalid_argument("the trapezoid rule is the one identity tap"); - if (problem.plasma_gated && - (!problem.plasma || problem.envelope_plasma || problem.general_modal)) + if (problem.plasma.gated && + (!problem.plasma.enabled || problem.plasma.envelope || problem.general.enabled)) throw std::invalid_argument( "the gated plasma benchmark needs a field plasma column off the general route"); - if (problem.plasma_band) { - if (!problem.plasma || problem.envelope_plasma || - problem.family == FieldFamily::envelope) + if (problem.plasma.band) { + if (!problem.plasma.enabled || problem.plasma.envelope || + problem.dims.family == FieldFamily::envelope) throw std::invalid_argument( "the band integration rule needs a field-grid plasma column"); - if (problem.n_to <= problem.plasma_half_width()) + if (problem.dims.n_to <= problem.plasma.half_width()) throw std::invalid_argument( "the band integration rule's filter needs more than K samples"); } - if (problem.envelope_plasma) { - if (!problem.plasma || problem.family != FieldFamily::envelope || - problem.harmonic_factor != 0.0 || problem.raman_thg || problem.stash_plasma) + if (problem.plasma.envelope) { + if (!problem.plasma.enabled || problem.dims.family != FieldFamily::envelope || + problem.kerr.harmonic_factor != 0.0 || problem.raman.thg || + problem.plasma.stash) throw std::invalid_argument("envelope plasma requires separate envelope currents " "without THG or field stashing"); - const auto rule_size = - checked_product(problem.n_channels, 4 * 512, "envelope rules"); - require_extent(problem.envelope_rule_nodes.size(), rule_size, + const auto rule_size = checked_product(problem.plasma.n_channels, + std::size_t{4} * 512, "envelope rules"); + require_extent(problem.plasma.envelope_rule_nodes.size(), rule_size, "envelope_rule_nodes"); - require_extent(problem.envelope_rule_weights.size(), rule_size, + require_extent(problem.plasma.envelope_rule_weights.size(), rule_size, "envelope_rule_weights"); - for (std::size_t channel = 0; channel < problem.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem.plasma.n_channels; ++channel) for (std::size_t level = 0; level < 4; ++level) { const auto order = std::size_t{64} << level; - double sum = 0.0, previous = -1.0; + double sum = 0.0; + double previous = -1.0; for (std::size_t index = 0; index < 512; ++index) { const auto offset = (channel * 4 + level) * 512 + index; - const auto node = problem.envelope_rule_nodes[offset]; - const auto weight = problem.envelope_rule_weights[offset]; + const auto node = problem.plasma.envelope_rule_nodes[offset]; + const auto weight = problem.plasma.envelope_rule_weights[offset]; if (!std::isfinite(node) || !std::isfinite(weight) || (index < order ? !(node > previous && node < 1.0 && weight > 0.0) : node != 0.0 || weight != 0.0)) @@ -1458,14 +1499,15 @@ void require_general_problem(const Problem& problem) { throw std::invalid_argument( "envelope quadrature weights must integrate a constant"); } - require_extent(problem.envelope_drude_factor.size(), problem.n_omega, + require_extent(problem.plasma.envelope_drude_factor.size(), problem.dims.n_omega, "envelope_drude_factor"); - require_extent(problem.envelope_loss_factor.size(), problem.n_omega, + require_extent(problem.plasma.envelope_loss_factor.size(), problem.dims.n_omega, "envelope_loss_factor"); - for (std::size_t index = 0; index < problem.n_omega; ++index) { - const auto drude = problem.envelope_drude_factor[index]; - const auto loss = problem.envelope_loss_factor[index]; - const bool active = problem.active_mask[index] != 0 && problem.omega[index] > 0.0; + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + const auto drude = problem.plasma.envelope_drude_factor[index]; + const auto loss = problem.plasma.envelope_loss_factor[index]; + const bool active = + problem.grid.active_mask[index] != 0 && problem.grid.omega[index] > 0.0; if (!std::isfinite(drude) || !std::isfinite(loss.real()) || !std::isfinite(loss.imag()) || loss.real() != 0.0 || (active ? !(drude < 0.0 && loss.imag() < 0.0) @@ -1474,41 +1516,41 @@ void require_general_problem(const Problem& problem) { "factors and zero inactive bins"); } } - if (problem.raman) { - if (effective_node_pol != 1 && !problem.raman_tensor_capable) + if (problem.raman.enabled) { + if (effective_node_pol != 1 && !problem.raman.tensor_capable) throw std::invalid_argument( "vector Raman requires tensor-capable terms; found scalar_only"); - if (problem.n_raman_terms == 0) + if (problem.raman.n_terms == 0) throw std::invalid_argument("a Raman problem must contain at least one term"); - require_extent(problem.raman_tensor_weights.size(), - checked_product(problem.n_raman_terms, 2, "Raman tensor weights"), + require_extent(problem.raman.tensor_weights.size(), + checked_product(problem.raman.n_terms, 2, "Raman tensor weights"), "raman_tensor_weights"); - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) { - const double trace = problem.raman_tensor_weights[2 * term]; - const double traceless = problem.raman_tensor_weights[2 * term + 1]; + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) { + const double trace = problem.raman.tensor_weights[2 * term]; + const double traceless = problem.raman.tensor_weights[2 * term + 1]; if (!((trace == 1.0 && traceless == 0.0) || (trace == 0.25 && traceless == 0.75))) throw std::invalid_argument( "Raman tensor weights must be isotropic or rank two"); } - const auto doubled = checked_product(2, problem.n_to, "Raman doubled grid"); - const auto raman_rows = - checked_product(problem.node_chunk, problem.raman_drives(), "Raman drive rows"); + const auto doubled = checked_product(2, problem.dims.n_to, "Raman doubled grid"); + const auto raman_rows = checked_product(problem.dims.node_chunk, + problem.raman_drives(), "Raman drive rows"); require_allocation(checked_product(raman_rows, doubled, "Raman drive"), sizeof(double), "Raman drive bytes"); - require_extent(problem.raman_undamped.size(), - checked_product(problem.n_raman_terms, doubled, "raman_undamped"), + require_extent(problem.raman.undamped.size(), + checked_product(problem.raman.n_terms, doubled, "raman_undamped"), "raman_undamped"); - if (!(problem.raman_fraction > 0.0) || !std::isfinite(problem.raman_fraction) || - !(problem.dt_fine > 0.0) || !std::isfinite(problem.dt_fine) || - !(problem.raman_epsilon0 > 0.0) || !std::isfinite(problem.raman_epsilon0)) + if (!(problem.raman.fraction > 0.0) || !std::isfinite(problem.raman.fraction) || + !(problem.grid.dt_fine > 0.0) || !std::isfinite(problem.grid.dt_fine) || + !(problem.raman.epsilon0 > 0.0) || !std::isfinite(problem.raman.epsilon0)) throw std::invalid_argument( "Raman fraction, dt_fine and epsilon0 must be positive and finite"); - if (!std::all_of(problem.raman_undamped.begin(), problem.raman_undamped.end(), - [](const double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(problem.raman.undamped, + [](const double value) { return std::isfinite(value); })) throw std::invalid_argument("raman_undamped must contain only finite values"); - } else if (problem.raman_thg || problem.n_raman_terms != 0 || - !problem.raman_undamped.empty() || !problem.raman_tensor_weights.empty() || - problem.raman_tensor_capable) { + } else if (problem.raman.thg || problem.raman.n_terms != 0 || + !problem.raman.undamped.empty() || !problem.raman.tensor_weights.empty() || + problem.raman.tensor_capable) { throw std::invalid_argument("Raman data were supplied while Raman is disabled"); } } @@ -1629,7 +1671,7 @@ void accept_update(const ExecutionSpace& execution, const std::size_t count, update.error_sum += squared_abs(one_residual); }, Kokkos::Sum(sums)); - Kokkos::single(Kokkos::PerTeam(team), [&]() { + Kokkos::single(Kokkos::PerTeam(team), [&] { values.old_partial[slot] = sums.old_sum; values.new_partial[slot] = sums.new_sum; values.error_partial[slot] = sums.error_sum; @@ -1669,9 +1711,13 @@ struct EntryBlock { double magnitude[stage_block], inverse[stage_block]; double noise_cosine[stage_block], noise_sine[stage_block]; - EntryFactors at(const std::size_t bin) const { - return {cosine[bin], sine[bin], magnitude[bin], - inverse[bin], noise_cosine[bin], noise_sine[bin]}; + [[nodiscard]] EntryFactors at(const std::size_t bin) const { + return {.cosine = cosine[bin], + .sine = sine[bin], + .magnitude = magnitude[bin], + .inverse = inverse[bin], + .noise_cosine = noise_cosine[bin], + .noise_sine = noise_sine[bin]}; } }; @@ -1971,8 +2017,10 @@ struct StageTerms { const double error_weight, const std::size_t column, const std::size_t first, const std::size_t count) const { - double real[stage_block], imag[stage_block]; - double source_real[stage_block], source_imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; + double source_real[stage_block]; + double source_imag[stage_block]; for (std::size_t bin = 0; bin < count; ++bin) { real[bin] = 0.0; imag[bin] = 0.0; @@ -2048,8 +2096,10 @@ struct StageTerms { combine_block(const std::size_t stage, const double dz, const std::size_t column, const std::size_t first, const std::size_t count, const EntryBlock& factors, const bool store_whole) const { - double real[stage_block], imag[stage_block]; - double sum_real[stage_block], sum_imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; + double sum_real[stage_block]; + double sum_imag[stage_block]; Complex* inverse = &stage_inverse(column, first); for (std::size_t bin = 0; bin < count; ++bin) { if constexpr (Mode == EntryPhase::repeated) { @@ -2131,45 +2181,47 @@ class GeneralWorkspace { explicit GeneralWorkspace(Problem problem) : problem_(std::move(problem)) { require_general_problem(problem_); require_cpu_execution(problem_.cpu_execution); - identity_crop_ = problem_.crop_index.size() == problem_.n_omega; - for (std::size_t index = 0; identity_crop_ && index < problem_.n_omega; ++index) - identity_crop_ = problem_.crop_index[index] == index; - one_mode_products_ = one_mode_products_enabled && - problem_.family == FieldFamily::field_real && - problem_.n_modes == 1 && problem_.n_pol_state == 1 && - problem_.n_pol_transform == 1 && problem_.n_nodes > 1 && - problem_.node_chunk == problem_.n_nodes; + identity_crop_ = problem_.grid.crop_index.size() == problem_.dims.n_omega; + for (std::size_t index = 0; identity_crop_ && index < problem_.dims.n_omega; + ++index) + identity_crop_ = problem_.grid.crop_index[index] == index; + one_mode_products_ = + one_mode_products_enabled && problem_.dims.family == FieldFamily::field_real && + problem_.dims.n_modes == 1 && problem_.dims.n_pol_state == 1 && + problem_.dims.n_pol_transform == 1 && problem_.dims.n_nodes > 1 && + problem_.dims.node_chunk == problem_.dims.n_nodes; compact_ = compact_problem(problem_); if (compact_) for (auto* pool : {&kernel_pool_, &fft_pool_, &raman_pool_, &raman_tail_pool_}) pool->compact(); - execution_space_ = - kernel_pool_.get(CpuOperation::stage, problem_.n_columns() * problem_.n_omega); - const auto nodes = problem_.node_chunk; + execution_space_ = kernel_pool_.get(CpuOperation::stage, + problem_.n_columns() * problem_.dims.n_omega); + const auto nodes = problem_.dims.node_chunk; const auto response_elements = - nodes * problem_.effective_node_pol() * problem_.n_to; + nodes * problem_.effective_node_pol() * problem_.dims.n_to; response_execution_space_ = - kernel_pool_.get(CpuOperation::response, response_elements, problem_.n_to); + kernel_pool_.get(CpuOperation::response, response_elements, problem_.dims.n_to); // On one thread the vendor GEMM's serial kernels are as fast. one_mode_products_ = one_mode_products_ && response_execution_space_.concurrency() > 1; - const auto tail = problem_.n_nodes % nodes; + const auto tail = problem_.dims.n_nodes % nodes; response_tail_execution_space_ = kernel_pool_.get( - CpuOperation::response, tail * problem_.effective_node_pol() * problem_.n_to, - problem_.n_to); - const auto dense_elements = response_elements * problem_.n_modes; - const auto tail_dense_elements = - tail * problem_.effective_node_pol() * problem_.n_to * problem_.n_modes; + CpuOperation::response, + tail * problem_.effective_node_pol() * problem_.dims.n_to, problem_.dims.n_to); + const auto dense_elements = response_elements * problem_.dims.n_modes; + const auto tail_dense_elements = tail * problem_.effective_node_pol() * + problem_.dims.n_to * problem_.dims.n_modes; transform_execution_space_ = kernel_pool_.get(CpuOperation::dense, dense_elements); transform_tail_execution_space_ = kernel_pool_.get(CpuOperation::dense, tail_dense_elements); temporal_execution_space_ = - kernel_pool_.get(CpuOperation::copy, problem_.n_columns() * problem_.n_to); + kernel_pool_.get(CpuOperation::copy, problem_.n_columns() * problem_.dims.n_to); // Packing copies only matrix entries, and splitting copies only output samples; // neither repeats the dense transform's inner product. - const auto matrix_elements = nodes * problem_.n_pol_transform * problem_.n_modes; + const auto matrix_elements = + nodes * problem_.dims.n_pol_transform * problem_.dims.n_modes; const auto tail_matrix_elements = - tail * problem_.n_pol_transform * problem_.n_modes; + tail * problem_.dims.n_pol_transform * problem_.dims.n_modes; matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, matrix_elements); tail_matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, tail_matrix_elements); @@ -2177,14 +2229,16 @@ class GeneralWorkspace { kernel_pool_.get(CpuOperation::copy, 2 * matrix_elements); tail_stacked_matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, 2 * tail_matrix_elements); - node_spectral_space_ = kernel_pool_.get( - CpuOperation::copy, nodes * problem_.effective_node_pol() * problem_.n_omega); - flow_execution_space_ = - kernel_pool_.get(CpuOperation::flow, problem_.flow_active * problem_.n_modes * - problem_.n_modes * problem_.n_modes); - if (!problem_.spatial_absorber_matrix.empty()) + node_spectral_space_ = + kernel_pool_.get(CpuOperation::copy, + nodes * problem_.effective_node_pol() * problem_.dims.n_omega); + flow_execution_space_ = kernel_pool_.get( + CpuOperation::flow, problem_.flow.active * problem_.dims.n_modes * + problem_.dims.n_modes * problem_.dims.n_modes); + if (!problem_.spatial.absorber_matrix.empty()) absorber_execution_space_ = kernel_pool_.get( - CpuOperation::dense, problem_.n_columns() * problem_.n_to * problem_.n_modes); + CpuOperation::dense, + problem_.n_columns() * problem_.dims.n_to * problem_.dims.n_modes); // solprop SEP 46: the workspace's own FFT planning, wisdom isolated from the rest // of the process in a calibration's modes. const FftPlanningScope planning(problem_.fft_planning); @@ -2340,8 +2394,9 @@ class GeneralWorkspace { require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - if constexpr (!Kokkos::SpaceAccessibility::accessible) + if constexpr (!static_cast( + Kokkos::SpaceAccessibility::accessible)) return nullptr; execution_space_.fence("solprop_native host_field"); return current_.data(); @@ -2386,12 +2441,15 @@ class GeneralWorkspace { } std::map raman_fft_tasks() const { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int full = 0, tail = 0, kernel = 0, products = 0; + const int full = 0; + const int tail = 0; + const int kernel = 0; + const int products = 0; #else - const int full = raman_execution_space_.concurrency(), - tail = raman_tail_execution_space_.concurrency(), - kernel = raman_kernel_execution_space_.concurrency(), - products = raman_product_execution_space_.concurrency(); + const int full = raman_execution_space_.concurrency(); + const int tail = raman_tail_execution_space_.concurrency(); + const int kernel = raman_kernel_execution_space_.concurrency(); + const int products = raman_product_execution_space_.concurrency(); #endif // `products` is nonzero when the modal products replace the node drives. return { @@ -2413,7 +2471,7 @@ class GeneralWorkspace { double quadrature_peak(); std::string synthesis_provider() const; std::string projection_provider() const; - static std::string gemm_shape_policy(); + static std::string_view gemm_shape_policy(); std::size_t response_team_size() const noexcept; bool fsal_ready() const noexcept { return has_recycled_; } std::size_t rk_source_evaluations() const noexcept { return rk_source_evaluations_; } @@ -2424,12 +2482,12 @@ class GeneralWorkspace { // NVCC extended device lambdas require their enclosing member to be public. // GeneralWorkspace is translation-unit-local, so this does not widen the API. std::size_t fine_spectrum_size() const { - return problem_.family == FieldFamily::field_real ? problem_.n_to / 2 + 1 - : problem_.n_to; + return problem_.dims.family == FieldFamily::field_real ? problem_.dims.n_to / 2 + 1 + : problem_.dims.n_to; } std::size_t projected_spectrum_size() const { - return problem_.family == FieldFamily::envelope ? problem_.n_to - : problem_.n_to / 2 + 1; + return problem_.dims.family == FieldFamily::envelope ? problem_.dims.n_to + : problem_.dims.n_to / 2 + 1; } void require_open() const { if (closed_) @@ -2486,17 +2544,20 @@ class GeneralWorkspace { // Free space and the hybrid planar kind (`Problem::separable_geometry`). TransverseLayout transverse_layout() const noexcept { const std::size_t plane = transverse_.extent(1) * transverse_.extent(2); - return problem_.slab_geometry() ? TransverseLayout{problem_.n_p, true, plane} - : TransverseLayout{problem_.n_x, false, plane}; + return problem_.slab_geometry() ? TransverseLayout{.inner = problem_.spatial.n_p, + .channel_fastest = true, + .plane = plane} + : TransverseLayout{.inner = problem_.spatial.n_x, + .channel_fastest = false, + .plane = plane}; } RegisterIndex register_index() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr bool plane = false; -#else - constexpr bool plane = true; -#endif - return {transverse_layout(), problem_.n_pol_state, problem_.n_omega, plane, - problem_.n_pol_state == 1}; + constexpr bool plane = build::host; + return {.layout = transverse_layout(), + .n_pol = problem_.dims.n_pol_state, + .n = problem_.dims.n_omega, + .plane = plane, + .scalar = problem_.dims.n_pol_state == 1}; } // A stage's scratch positions: the volume route's own order on a device, or the // scratch's (`register_index`). Every other pass over the scratch forms its own input @@ -2510,15 +2571,15 @@ class GeneralWorkspace { } // The unscaled transverse DFT's sample count: the plane's, or the slab's y axis. double inverse_transverse_count() const noexcept { - return 1.0 / static_cast(problem_.slab_geometry() ? problem_.n_y - : problem_.n_modes); + return 1.0 / static_cast(problem_.slab_geometry() ? problem_.spatial.n_y + : problem_.dims.n_modes); } std::size_t transverse_units() const noexcept { - return problem_.slab_geometry() ? problem_.n_y : problem_.n_nodes; + return problem_.slab_geometry() ? problem_.spatial.n_y : problem_.dims.n_nodes; } std::size_t transverse_units_per_chunk() const noexcept { - return problem_.slab_geometry() ? problem_.node_chunk / problem_.n_x - : problem_.node_chunk; + return problem_.slab_geometry() ? problem_.dims.node_chunk / problem_.spatial.n_x + : problem_.dims.node_chunk; } TransverseChunk transverse_chunk(std::size_t first_unit) const noexcept; void transverse_forward(); @@ -2631,15 +2692,15 @@ class GeneralWorkspace { const EnvelopeStream* stream = nullptr, const FieldCurrentStream* field_stream = nullptr); bool field_plasma() const noexcept { - return problem_.plasma && !problem_.envelope_plasma; + return problem_.plasma.enabled && !problem_.plasma.envelope; } - bool held_plasma() const noexcept { return field_plasma() && !problem_.plasma_gated; } + bool held_plasma() const noexcept { return field_plasma() && !problem_.plasma.gated; } // solprop's private gated benchmark: the current's running integral, gated by the // time window, joins the local polarisation before it is projected. - bool gated_plasma() const noexcept { return field_plasma() && problem_.plasma_gated; } + bool gated_plasma() const noexcept { return field_plasma() && problem_.plasma.gated; } bool held_radial() const noexcept { return held_plasma() && !problem_.separable_geometry() && - !problem_.spatial_spectral_weights.empty(); + !problem_.spatial.spectral_weights.empty(); } // Whether a device takes a modal stage's forward transforms -- the projection's // (fine and coarse), the in-stage absorber's and the held currents', each a real row @@ -2666,16 +2727,17 @@ class GeneralWorkspace { if (!host_forward_pack() || compact_) return false; #endif - return problem_.family == FieldFamily::field_real && - !problem_.separable_geometry() && !problem_.general_modal && - !problem_.dispersive && !problem_.envelope_plasma && !held_radial() && - (problem_.in_stage_absorber || held_plasma() || - !problem_.projection_coarse_matrix.empty()); + return problem_.dims.family == FieldFamily::field_real && + !problem_.separable_geometry() && !problem_.general.enabled && + !problem_.dispersive.enabled && !problem_.plasma.envelope && + !held_radial() && + (problem_.absorber.in_stage || held_plasma() || + !problem_.transform.projection_coarse_matrix.empty()); } static bool host_forward_pack() noexcept { static const bool enabled = [] { const char* text = std::getenv("SOLPROP_NATIVE_FORWARD_PACK"); - return text == nullptr || std::string(text) != "0"; + return text == nullptr || std::string_view(text) != "0"; }(); return enabled; } @@ -2686,16 +2748,14 @@ class GeneralWorkspace { void held_transform(bool coarse_only = false, bool transformed = false); void finish_quadrature(); bool deferred_quadrature_eligible() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else + if constexpr (build::device) + return false; // The validated seven-stage FSAL tableau identifies the ordinary DOPRI5 // route without another seam field. Other consumers retain eager spectra. - return problem_.first_same_as_last && problem_.n_stages == 7 && - !problem_.low_storage && !problem_.separable_geometry() && !general_ && - !flow_ && !dispersive_ && !problem_.envelope_plasma && !held_radial() && - !problem_.projection_coarse_matrix.empty(); -#endif + return problem_.stepper.first_same_as_last && problem_.stepper.n_stages == 7 && + !problem_.stepper.low_storage && !problem_.separable_geometry() && + !general_ && !flow_ && !dispersive_ && !problem_.plasma.envelope && + !held_radial() && !problem_.transform.projection_coarse_matrix.empty(); } void held_radial_projection(std::size_t first_node, std::size_t node_count); void envelope_cycle(std::size_t channel, std::size_t node_count, @@ -2721,7 +2781,7 @@ class GeneralWorkspace { const RealView2& output, bool diagnose = false, double preionisation = 0.0); void form_absorber(); - void split_step(double dz, const ComplexView2& values); + void split_step(double dz, const ComplexView2& advanced); void form_split_factor(double dz); void split_columns(double dz, std::size_t active_rows = 0, double parseval = 0.0, std::size_t first_column = 0); @@ -2731,7 +2791,7 @@ class GeneralWorkspace { double power_scale); void separable_to_time(const TransverseChunk& chunk); void separable_to_frequency(const TransverseChunk& chunk); - void separable_split_step(double dz, const ComplexView2& values); + void separable_split_step(double dz, const ComplexView2& advanced); void mask(const ComplexView2& values); bool use_packed_radial_blas() const noexcept; // A square radial analytic map synthesises the retained spectrum, before padding, @@ -2776,9 +2836,10 @@ class GeneralWorkspace { const ExecutionSpace& matrix_copy_space_for(std::size_t nodes, bool stacked = false) const { if (stacked) - return nodes == problem_.node_chunk ? stacked_matrix_copy_space_ - : tail_stacked_matrix_copy_space_; - return nodes == problem_.node_chunk ? matrix_copy_space_ : tail_matrix_copy_space_; + return nodes == problem_.dims.node_chunk ? stacked_matrix_copy_space_ + : tail_stacked_matrix_copy_space_; + return nodes == problem_.dims.node_chunk ? matrix_copy_space_ + : tail_matrix_copy_space_; } ExecutionSpace fft_execution_space_; ExecutionSpace temporal_execution_space_, flow_execution_space_, @@ -2792,12 +2853,12 @@ class GeneralWorkspace { raman_kernel_execution_space_; ExecutionSpace raman_product_execution_space_, raman_product_space_; const ExecutionSpace& response_space_for(std::size_t nodes) const { - return nodes == problem_.node_chunk ? response_execution_space_ - : response_tail_execution_space_; + return nodes == problem_.dims.node_chunk ? response_execution_space_ + : response_tail_execution_space_; } const ExecutionSpace& transform_space_for(std::size_t nodes) const { - return nodes == problem_.node_chunk ? transform_execution_space_ - : transform_tail_execution_space_; + return nodes == problem_.dims.node_chunk ? transform_execution_space_ + : transform_tail_execution_space_; } bool closed_{}; bool initialized_{}; @@ -2821,7 +2882,7 @@ class GeneralWorkspace { // `finish_pending_` names the stage's finish still to run there. bool defer_finish_{}; bool finish_pending_{}; - // `problem_.crop_index` is the identity (`StageTerms::identity_crop`). + // `problem_.grid.crop_index` is the identity (`StageTerms::identity_crop`). bool identity_crop_{}; // The bound trial's tapers on the host, and the one the scaled transforms hold // (`scale_transform`). @@ -2997,7 +3058,8 @@ class GeneralWorkspace { bool band_tiled_{band_tiled(thread_policy())}; // Each column's norm reductions on a device are this many teams (`reduction_slots`): // the norm partials hold `columns * reduction_slots_` slots, reduced on the host. - std::size_t reduction_slots_{reduction_slots(problem_.n_columns(), problem_.n_omega)}; + std::size_t reduction_slots_{ + reduction_slots(problem_.n_columns(), problem_.dims.n_omega)}; std::size_t band_tile_{}; std::size_t band_tile_count_{}; int band_tile_threads_{band_tile_threads(thread_policy())}; @@ -3196,16 +3258,16 @@ class GeneralWorkspace { // A large crossing's pinned chunks, once one has been made (`crossing.hpp`). mutable crossing::Ring crossing_ring_; [[nodiscard]] bool monitoring_split() const noexcept { - return monitor_bound_ && problem_.split_step_absorber; + return monitor_bound_ && problem_.absorber.split_step; } // Whether the stage adds the in-stage absorber's term. A quiet temporal collar is // left out on the modal routes, as solprop's reference leaves it out, unless a // real-space collar shares the term or the route is the general modal one or a // coupled flow, which keep it (solprop SEP 21). [[nodiscard]] bool absorbing() const noexcept { - return problem_.in_stage_absorber && - !(absorber_quiet_ && !general_ && !problem_.coupled_flow && - problem_.spatial_absorber_matrix.empty()); + return problem_.absorber.in_stage && + !(absorber_quiet_ && !general_ && !problem_.flow.enabled && + problem_.spatial.absorber_matrix.empty()); } // solprop's linear.coupled_flow; null otherwise. std::unique_ptr flow_; @@ -3239,30 +3301,25 @@ class GeneralWorkspace { // 36 nodes, 10240 samples) took 71 us in cuBLAS and its synthesis 27 us; at 256 // radial modes the vendor GEMM runs near the FP64 peak. bool GeneralWorkspace::hybrid_shape_eligible() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return problem_.n_modes >= 8 && problem_.n_nodes >= 8 && problem_.n_to >= 1024; -#else - return true; -#endif + return build::host || (problem_.dims.n_modes >= 8 && problem_.dims.n_nodes >= 8 && + problem_.dims.n_to >= 1024); } bool GeneralWorkspace::use_packed_radial_blas() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else - const auto provider = compiled_gemm_provider(); + if constexpr (build::device) + return false; // Two packed work arrays add two complex-equivalent time columns to the // shared conservative twenty-column inventory. Decide before allocation. - constexpr std::size_t max_samples = 64 * 1024 * 1024 / (22 * sizeof(Complex)); - return (provider == GemmProvider::hybrid || - provider == GemmProvider::kokkosblas_tpl) && - !problem_.separable_geometry() && !problem_.spatial_spectral_weights.empty() && - problem_.n_modes == problem_.n_nodes && - problem_.node_chunk == problem_.n_nodes && problem_.n_pol_state == 1 && - problem_.n_pol_transform == 1 && problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && problem_.n_nodes >= 256 && - problem_.n_to >= 256 && problem_.n_nodes <= max_samples / problem_.n_to; -#endif + constexpr std::size_t max_samples = + std::size_t{64} * 1024 * 1024 / (22 * sizeof(Complex)); + return !problem_.separable_geometry() && !problem_.spatial.spectral_weights.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && + problem_.dims.node_chunk == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_nodes >= 256 && problem_.dims.n_to >= 256 && + problem_.dims.n_nodes <= max_samples / problem_.dims.n_to; } // The retained-spectrum radial synthesis, on a host and a device. A real static @@ -3275,21 +3332,20 @@ bool GeneralWorkspace::use_packed_radial_blas() const noexcept { // modal time field, and a dispersive response its own synthesis, so they keep the // time-domain product; skinny capillary maps would multiply the transforms. bool GeneralWorkspace::retained_synthesis() const noexcept { - return compiled_gemm_provider() != GemmProvider::custom && - problem_.family == FieldFamily::field_analytic && - !problem_.spatial_spectral_weights.empty() && !problem_.separable_geometry() && - !problem_.general_modal && !problem_.dispersive && - !problem_.in_stage_absorber && !problem_.raman && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes > 1 && - problem_.n_pol_state == 1 && problem_.n_pol_transform == 1 && - problem_.n_omega < problem_.n_to; + return problem_.dims.family == FieldFamily::field_analytic && + !problem_.spatial.spectral_weights.empty() && !problem_.separable_geometry() && + !problem_.general.enabled && !problem_.dispersive.enabled && + !problem_.absorber.in_stage && !problem_.raman.enabled && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes > 1 && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.dims.n_omega < problem_.dims.n_to; } void GeneralWorkspace::synthesise_retained(const std::size_t first_node, const std::size_t node_count) { Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.retained"); const auto crop = crop_index_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; // The coarse modal spectra once per evaluation, at its first chunk (the stage core // and the statistics' synthesis both start at node zero). Rows run in parallel at // the modal copies' width (`temporal_execution_space_`), each a contiguous pass. @@ -3300,7 +3356,7 @@ void GeneralWorkspace::synthesise_retained(const std::size_t first_node, // A device takes a thread per element. launch::parallel_for( "solprop_native.transform.retained_gather", - Range2(execution_space_, {0, 0}, {problem_.n_modes, n}), + Range2(execution_space_, {0, 0}, {problem_.dims.n_modes, n}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t index) { modal(mode, index) = fine(mode, crop(index)); }); @@ -3308,7 +3364,7 @@ void GeneralWorkspace::synthesise_retained(const std::size_t first_node, launch::parallel_for( "solprop_native.transform.retained_gather", Kokkos::RangePolicy(temporal_execution_space_, 0, - problem_.n_modes), + problem_.dims.n_modes), KOKKOS_LAMBDA(const std::size_t mode) { for (std::size_t index = 0; index < n; ++index) modal(mode, index) = fine(mode, crop(index)); @@ -3318,10 +3374,11 @@ void GeneralWorkspace::synthesise_retained(const std::size_t first_node, using Interleaved = Kokkos::View>; static_assert(sizeof(Complex) == 2 * sizeof(double)); - const Interleaved synthesis(scaled_synthesis_.data() + first_node * problem_.n_modes, - node_count, problem_.n_modes); + const Interleaved synthesis(scaled_synthesis_.data() + + first_node * problem_.dims.n_modes, + node_count, problem_.dims.n_modes); const Interleaved input(reinterpret_cast(retained_modal_.data()), - problem_.n_modes, 2 * n); + problem_.dims.n_modes, 2 * n); const Interleaved output(reinterpret_cast(retained_node_.data()), node_count, 2 * n); engine_gemm(transform_space_for(node_count), synthesis, input, output, 0.0); @@ -3354,7 +3411,7 @@ void GeneralWorkspace::synthesise_retained(const std::size_t first_node, padded(row, crop(index)) = node(row, index); }); #endif - const bool tail = node_count != problem_.node_chunk; + const bool tail = node_count != problem_.dims.node_chunk; const auto rows = std::make_pair(std::size_t{0}, node_count); graph::execute(tail ? *retained_tail_inverse_plan_ : *retained_inverse_plan_, Kokkos::subview(modal_complex_, rows, Kokkos::ALL()), @@ -3368,8 +3425,8 @@ void GeneralWorkspace::packed_radial_gemm(const RealView2& matrix, const ComplexView2& destination) { const auto input = radial_blas_input_; const auto output = radial_blas_output_; - const auto rows = problem_.n_nodes; - const auto times = problem_.n_to; + const auto rows = problem_.dims.n_nodes; + const auto times = problem_.dims.n_to; Kokkos::Profiling::pushRegion("solprop_native.radial.packed_real_gemm"); launch::parallel_for( "solprop_native.radial.pack", @@ -3395,33 +3452,28 @@ bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { // The hybrid planar kind contracts its small x rule per y row natively. if (problem_.separable_geometry()) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; - if (problem_.n_pol_state != 1) + if (problem_.dims.n_pol_state != 1) return false; - if (problem_.family == FieldFamily::envelope) { -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - if (provider != GemmProvider::kokkosblas_tpl && provider != GemmProvider::hybrid) - return false; -#else +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.dims.family == FieldFamily::envelope) return false; #endif - } else if (problem_.family != FieldFamily::field_real) { + if (problem_.dims.family != FieldFamily::envelope && + problem_.dims.family != FieldFamily::field_real) return false; - } #if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) // Spatial envelope matrices can be large even with a short temporal grid. - if (!problem_.spatial_spectral_weights.empty() && - problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes >= 256 && - problem_.n_to >= 256) + if (!problem_.spatial.spectral_weights.empty() && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes >= 256 && + problem_.dims.n_to >= 256) return true; #endif - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } // An analytic field's or an envelope's synthesis is a real matrix times complex @@ -3433,23 +3485,18 @@ bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { // (`hybrid_shape_eligible`): there the custom synthesis kernel cost 4.6 times the // projection's GEMM at 512 radial nodes (an AMD CDNA 3 device, 2026-10-06). bool GeneralWorkspace::use_interleaved_synthesis() const noexcept { - if (problem_.family == FieldFamily::field_real || problem_.separable_geometry() || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) + if (problem_.dims.family == FieldFamily::field_real || + problem_.separable_geometry() || problem_.dims.n_pol_state != 1 || + problem_.dims.n_pol_transform != 1) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_interleaved_projection() const noexcept { - if (problem_.family != FieldFamily::envelope || problem_.separable_geometry() || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) - return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) + if (problem_.dims.family != FieldFamily::envelope || problem_.separable_geometry() || + problem_.dims.n_pol_state != 1 || problem_.dims.n_pol_transform != 1) return false; - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_blas_for_projection() const noexcept { @@ -3458,35 +3505,29 @@ bool GeneralWorkspace::use_blas_for_projection() const noexcept { // The hybrid planar kind contracts its small x rule per y row natively. if (problem_.separable_geometry()) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) + if (problem_.dims.n_pol_state != 1) return false; - if (problem_.n_pol_state != 1) - return false; - if (problem_.family == FieldFamily::envelope) { -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - if (provider != GemmProvider::kokkosblas_tpl && provider != GemmProvider::hybrid) - return false; -#else +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.dims.family == FieldFamily::envelope) return false; #endif - } #if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) // Spatial envelope matrices can be large even with a short temporal grid. - if (!problem_.spatial_spectral_weights.empty() && - problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes >= 256 && - problem_.n_to >= 256) + if (!problem_.spatial.spectral_weights.empty() && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes >= 256 && + problem_.dims.n_to >= 256) return true; #endif - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_chunked_blas_for_synthesis() const noexcept { - return problem_.node_chunk < problem_.n_nodes && use_blas_for_synthesis(); + return problem_.dims.node_chunk < problem_.dims.n_nodes && use_blas_for_synthesis(); } // One node, one mode and one scalar component: the synthesis and the projection are @@ -3496,20 +3537,19 @@ bool GeneralWorkspace::use_chunked_blas_for_synthesis() const noexcept { // `(s * taper) * x` and `(p / taper) * y`. // The host's columns only: a device keeps its own column kernels. bool GeneralWorkspace::folds_single_node() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else + if constexpr (build::device) + return false; return !general_ && !dispersive_ && !held_radial() && !gated_plasma() && - !problem_.separable_geometry() && problem_.n_modes == 1 && - problem_.n_nodes == 1 && problem_.node_chunk == 1 && - problem_.n_pol_state == 1 && problem_.n_pol_transform == 1 && - problem_.projection_coarse_matrix.empty() && !problem_.raman && - !problem_.envelope_plasma && radial_blas_input_.empty(); -#endif + !problem_.separable_geometry() && problem_.dims.n_modes == 1 && + problem_.dims.n_nodes == 1 && problem_.dims.node_chunk == 1 && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.transform.projection_coarse_matrix.empty() && + !problem_.raman.enabled && !problem_.plasma.envelope && + radial_blas_input_.empty(); } bool GeneralWorkspace::use_chunked_blas_for_projection() const noexcept { - return problem_.node_chunk < problem_.n_nodes && use_blas_for_projection(); + return problem_.dims.node_chunk < problem_.dims.n_nodes && use_blas_for_projection(); } std::string GeneralWorkspace::synthesis_provider() const { @@ -3540,7 +3580,7 @@ std::string GeneralWorkspace::projection_provider() const { return std::string(use_blas_for_projection() ? provider : std::string_view("custom")); } -std::string GeneralWorkspace::gemm_shape_policy() { +std::string_view GeneralWorkspace::gemm_shape_policy() { return "host radial scalar envelope uses packed real BLAS for whole transforms " "with n_nodes/n_to >= 256 and a 22-column inventory within 64 MiB; " "device radial scalar envelope uses vendor complex BLAS with " @@ -3561,104 +3601,111 @@ std::string GeneralWorkspace::gemm_shape_policy() { std::size_t GeneralWorkspace::response_team_size() const noexcept { constexpr auto configured = static_cast(SOLPROP_NATIVE_DEVICE_TEAM_SIZE); -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA // Measured on a Hopper GPU, the crossover is sharp: 4,096-sample fibre problems // favour teams of 256, while 8,192 samples and more favour 512. Keep an explicitly // configured smaller team intact. - if (problem_.n_to < 8192) + if (problem_.dims.n_to < 8192) return std::min(configured, std::size_t{256}); #endif return configured; } void GeneralWorkspace::allocate() { - if (problem_.low_storage - ? problem_.low_storage_flows > 2 || problem_.coupled_flow || - (!problem_.low_storage_stage_register && !problem_.free_geometry()) - : problem_.low_storage_flows != 0 || !problem_.low_storage_stage_register) + if (problem_.stepper.low_storage + ? problem_.stepper.low_storage_flows > 2 || problem_.flow.enabled || + (!problem_.stepper.low_storage_stage_register && + !problem_.free_geometry()) + : problem_.stepper.low_storage_flows != 0 || + !problem_.stepper.low_storage_stage_register) throw std::invalid_argument( "the low-storage march holds 0 to 2 flows beside no coupled flow, and elides " "its stage register on free space alone"); // The exchange fold of an in-stage absorber re-forms its input from the stage // register, and a separable fold applies one flow volume, a constant operator's // table (solprop SEP 24 section 4.3): neither has a meaning to fall back on. - if (problem_.low_storage && problem_.in_stage_absorber && - !problem_.low_storage_stage_register) + if (problem_.stepper.low_storage && problem_.absorber.in_stage && + !problem_.stepper.low_storage_stage_register) throw std::invalid_argument("the low-storage march cannot elide its stage register " "beside an in-stage absorber"); - if (problem_.low_storage && problem_.separable_geometry() && - problem_.low_storage_flows > 1) + if (problem_.stepper.low_storage && problem_.separable_geometry() && + problem_.stepper.low_storage_flows > 1) throw std::invalid_argument( "the low-storage march holds at most one flow volume on " "a separable geometry"); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto chunk_rows = problem_.node_chunk * problem_.effective_node_pol(); + const auto n = problem_.dims.n_omega; + const auto chunk_rows = problem_.dims.node_chunk * problem_.effective_node_pol(); const auto transform_rows = - problem_.separable_geometry() ? 0 : problem_.n_nodes * problem_.n_pol_transform; + problem_.separable_geometry() + ? 0 + : problem_.dims.n_nodes * problem_.dims.n_pol_transform; // A planar chunk's temporal rows are its y rows' columns, not its nodes. - const auto temporal_columns = problem_.slab_geometry() - ? transverse_units_per_chunk() * problem_.n_p - : (problem_.free_geometry() ? chunk_rows : columns); + const auto temporal_columns = + problem_.slab_geometry() ? transverse_units_per_chunk() * problem_.spatial.n_p + : (problem_.free_geometry() ? chunk_rows : columns); separable_aliased_ = problem_.free_geometry() && !problem_.slab_geometry() && - !problem_.dispersive && temporal_columns == chunk_rows; + !problem_.dispersive.enabled && temporal_columns == chunk_rows; const auto statistic_nodes = - problem_.separable_geometry() ? problem_.node_chunk : problem_.n_modes; + problem_.separable_geometry() ? problem_.dims.node_chunk : problem_.dims.n_modes; // Spatial statistics reconstruct one chunk of physical nodes at a time. const auto statistics_rows = problem_.slab_geometry() ? chunk_rows : temporal_columns; if (problem_.separable_geometry()) { // Keeping transform axes trailing avoids Kokkos-FFT's per-execution // transpose allocations. The canonical state retains omega-contiguous columns. - transverse_ = - problem_.slab_geometry() - ? ComplexView3("solprop_native.slab.transverse", n, problem_.n_p, - problem_.n_y) - : ComplexView3("solprop_native.free.transverse", problem_.n_pol_state * n, - problem_.n_y, problem_.n_x); + transverse_ = problem_.slab_geometry() + ? ComplexView3("solprop_native.slab.transverse", n, + problem_.spatial.n_p, problem_.spatial.n_y) + : ComplexView3("solprop_native.free.transverse", + problem_.dims.n_pol_state * n, + problem_.spatial.n_y, problem_.spatial.n_x); spatial_.square = ComplexView("solprop_native.free.k_squared", n); spatial_.square_low = ComplexView("solprop_native.free.k_squared_low", n); spatial_.numerator = ComplexView("solprop_native.free.numerator", n); spatial_.clamped = ComplexView("solprop_native.free.clamped", n); - spatial_.transverse = RealView("solprop_native.free.kperp_sq", problem_.n_modes); + spatial_.transverse = + RealView("solprop_native.free.kperp_sq", problem_.dims.n_modes); spatial_.transverse_low = - RealView("solprop_native.free.kperp_sq_low", problem_.n_modes); + RealView("solprop_native.free.kperp_sq_low", problem_.dims.n_modes); spatial_.frame = RealView("solprop_native.free.frame", n); spatial_.spectral_rate = RealView("solprop_native.free.spectral_rate", n); spatial_.user_rate = RealView("solprop_native.free.user_rate", n); - spatial_.absorber = RealView("solprop_native.free.absorber", problem_.n_modes); + spatial_.absorber = RealView("solprop_native.free.absorber", problem_.dims.n_modes); spatial_.transverse_rate = - RealView("solprop_native.free.kperp_rate", problem_.n_modes); - spatial_.evanescent = problem_.spatial_evanescent_rate; + RealView("solprop_native.free.kperp_rate", problem_.dims.n_modes); + spatial_.evanescent = problem_.spatial.evanescent_rate; if (problem_.slab_geometry()) { - spatial_.kappa = ComplexView2("solprop_native.slab.kappa_sq", problem_.n_p, n); - spatial_.channels = problem_.n_p; + spatial_.kappa = + ComplexView2("solprop_native.slab.kappa_sq", problem_.spatial.n_p, n); + spatial_.channels = problem_.spatial.n_p; } flux_ = spatial_; - if (problem_.flux_closure) + if (problem_.flux.closure) flux_transverse_ = ComplexView3("solprop_native.flux.closure_transverse", transverse_.extent(0), transverse_.extent(1), transverse_.extent(2)); - } else if (!problem_.flux_k_squared.empty()) { + } else if (!problem_.flux.k_squared.empty()) { flux_.square = ComplexView("solprop_native.flux.k_squared", n); flux_.square_low = ComplexView("solprop_native.flux.k_squared_low", n); - flux_.transverse = RealView("solprop_native.flux.kperp_sq", problem_.n_modes); + flux_.transverse = RealView("solprop_native.flux.kperp_sq", problem_.dims.n_modes); flux_.transverse_low = - RealView("solprop_native.flux.kperp_sq_low", problem_.n_modes); + RealView("solprop_native.flux.kperp_sq_low", problem_.dims.n_modes); } current_ = ComplexView2("solprop_native.state.current", columns, n); - if (problem_.low_storage) { + if (problem_.stepper.low_storage) { // solprop SEP 24 section 6.7: the march holds its registers beside the state and // none of the Runge-Kutta views; the stage's input is formed in the route's own // scratch (`transverse_`, or the fine spectrum of the modal routes). register_accumulator_ = ComplexView2("solprop_native.low_storage.accumulator", columns, n); - if (problem_.low_storage_stage_register) + if (problem_.stepper.low_storage_stage_register) register_stage_ = ComplexView2("solprop_native.low_storage.stage", columns, n); low_storage_flows_ = ComplexView3("solprop_native.low_storage.flows", - problem_.low_storage_flows, columns, n); + problem_.stepper.low_storage_flows, columns, n); } else { advanced_ = ComplexView2("solprop_native.state.advanced", columns, n); - stages_ = ComplexView3("solprop_native.step.stages", problem_.n_stages, columns, n); + stages_ = ComplexView3("solprop_native.step.stages", problem_.stepper.n_stages, + columns, n); solution_ = ComplexView2("solprop_native.step.solution", columns, n); error_ = ComplexView2("solprop_native.step.error", columns, n); residual_ = ComplexView2("solprop_native.step.residual", columns, n); @@ -3668,66 +3715,67 @@ void GeneralWorkspace::allocate() { if (!problem_.separable_geometry()) stage_inverse_ = ComplexView2("solprop_native.step.stage_inverse", columns, n); } - if (problem_.modified_noise) + if (problem_.stepper.modified_noise) noise_ = ComplexView2("solprop_native.noise", columns, n); - if (!problem_.separable_geometry() && problem_.coupled_flow) + if (!problem_.separable_geometry() && problem_.flow.enabled) stage_phase_ = ComplexView2("solprop_native.step.stage_phase", columns, n); partials_ = RealView2("solprop_native.step.error_partials", 3, checked_product(columns, reduction_slots_, "norm partial slots")); - if (!problem_.projection_coarse_matrix.empty()) + if (!problem_.transform.projection_coarse_matrix.empty()) quadrature_partials_ = RealView2("solprop_native.transform.quadrature_partials", 2, columns); quadrature_scores_ = RealView("solprop_native.transform.quadrature_scores", 2); Kokkos::deep_copy(quadrature_scores_, Kokkos::Experimental::quiet_NaN_v); - if (!problem_.low_storage) + if (!problem_.stepper.low_storage) pinned_partials_ = Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, partials_); - if (problem_.weighted_controller) { + if (problem_.stepper.weighted_controller) { weighted_scale_ = RealView("solprop_native.step.weighted_scale", n); weighted_partials_ = RealView("solprop_native.step.weighted_partials", columns * reduction_slots_); } - if (!problem_.spatial_spectral_weights.empty()) { + if (!problem_.spatial.spectral_weights.empty()) { spatial_spectral_weights_ = - RealView("solprop_native.spatial.spectral_weights", problem_.n_modes); - spatial_on_axis_ = ComplexView("solprop_native.spatial.on_axis", problem_.n_modes); + RealView("solprop_native.spatial.spectral_weights", problem_.dims.n_modes); + spatial_on_axis_ = + ComplexView("solprop_native.spatial.on_axis", problem_.dims.n_modes); // One row per column for the energy and per node for the peaks and fluence, // which differ on the hybrid planar kind alone. spatial_statistics_partials_ = RealView2("solprop_native.spatial.statistics_partials", - std::max(problem_.n_modes, problem_.n_nodes), 3); + std::max(problem_.dims.n_modes, problem_.dims.n_nodes), 3); } - statistics_spectrum_ = - ComplexView2("solprop_native.statistics.spectrum", statistics_rows, problem_.n_t); - statistics_time_ = - ComplexView2("solprop_native.statistics.time", statistics_rows, problem_.n_t); + statistics_spectrum_ = ComplexView2("solprop_native.statistics.spectrum", + statistics_rows, problem_.dims.n_t); + statistics_time_ = ComplexView2("solprop_native.statistics.time", statistics_rows, + problem_.dims.n_t); statistics_power_weight_ = RealView2("solprop_native.statistics.power_weight", - problem_.spatial_spectral_weights.empty() ? columns : 0, n); + problem_.spatial.spectral_weights.empty() ? columns : 0, n); graph::deep_copy(execution_space_, statistics_power_weight_, 1.0); // A spatial run's rows hold the magnetic closure in time (solprop SEP 34) when a // local flux statistic is asked for. statistics_weighted_time_ = ComplexView2("solprop_native.statistics.weighted_time", - problem_.spatial_spectral_weights.empty() ? temporal_columns - : problem_.flux_closure ? statistics_rows + problem_.spatial.spectral_weights.empty() ? temporal_columns + : problem_.flux.closure ? statistics_rows : 0, - problem_.n_t); + problem_.dims.n_t); statistics_power_ = - RealView2("solprop_native.statistics.power", statistic_nodes, problem_.n_t); - if (problem_.flux_closure && !problem_.separable_geometry()) + RealView2("solprop_native.statistics.power", statistic_nodes, problem_.dims.n_t); + if (problem_.flux.closure && !problem_.separable_geometry()) statistics_magnitude_ = RealView2("solprop_native.statistics.flux_magnitude", - statistic_nodes, problem_.n_t); + statistic_nodes, problem_.dims.n_t); statistics_total_power_ = - RealView("solprop_native.statistics.total_power", problem_.n_t); + RealView("solprop_native.statistics.total_power", problem_.dims.n_t); statistics_on_axis_ = - RealView("solprop_native.statistics.on_axis", 2 * problem_.n_modes); + RealView("solprop_native.statistics.on_axis", 2 * problem_.dims.n_modes); const auto base_statistics = checked_sum( - checked_sum(problem_.n_t, + checked_sum(problem_.dims.n_t, checked_product(2, problem_.n_columns(), "statistics columns"), "statistics packed output"), - checked_product(3, problem_.n_modes, "statistics modes"), + checked_product(3, problem_.dims.n_modes, "statistics modes"), "statistics packed output"); statistics_packed_ = RealView("solprop_native.statistics.packed", @@ -3736,30 +3784,32 @@ void GeneralWorkspace::allocate() { active_mask_ = ByteView("solprop_native.mask.active", n); norm_mask_ = ByteView("solprop_native.mask.norm", n); omega_window_ = RealView("solprop_native.window.omega", n); - omega_ = RealView("solprop_native.statistics.omega", problem_.omega.size()); - time_window_ = RealView("solprop_native.window.time", problem_.n_to); - carrier_ = ComplexView("solprop_native.kerr.carrier", problem_.n_to); - phi_terms_ = ComplexView3("solprop_native.linear.phi_terms", - problem_.separable_geometry() ? 0 : problem_.n_phi_terms, - columns, n); + omega_ = RealView("solprop_native.statistics.omega", problem_.grid.omega.size()); + time_window_ = RealView("solprop_native.window.time", problem_.dims.n_to); + carrier_ = ComplexView("solprop_native.kerr.carrier", problem_.dims.n_to); + phi_terms_ = ComplexView3( + "solprop_native.linear.phi_terms", + problem_.separable_geometry() ? 0 : problem_.linear.n_phi_terms, columns, n); prefactor_terms_ = ComplexView3( "solprop_native.response.prefactor_terms", - problem_.separable_geometry() ? 0 : problem_.n_prefactor_terms, columns, n); - tableau_ = - RealView("solprop_native.step.tableau", problem_.n_stages * problem_.n_stages); + problem_.separable_geometry() ? 0 : problem_.linear.n_prefactor_terms, columns, + n); + tableau_ = RealView("solprop_native.step.tableau", + problem_.stepper.n_stages * problem_.stepper.n_stages); { // One device buffer for every per-stage scalar view, so that a trial's scalars // cross in one copy instead of one per view: on a device each small copy from // pageable memory is a transfer of its own. - const auto stages = problem_.n_stages; - const auto channels = problem_.plasma ? problem_.n_channels : 0; - const auto raman_terms = problem_.raman ? problem_.n_raman_terms : 0; - const auto raman_components = problem_.raman && problem_.n_raman_components > 1 - ? problem_.n_raman_components - : 0; - const std::array sizes{(stages + 1) * problem_.n_phi_terms, - stages * problem_.n_phi_terms, - stages * problem_.n_prefactor_terms, + const auto stages = problem_.stepper.n_stages; + const auto channels = problem_.plasma.enabled ? problem_.plasma.n_channels : 0; + const auto raman_terms = problem_.raman.enabled ? problem_.raman.n_terms : 0; + const auto raman_components = + problem_.raman.enabled && problem_.raman.n_components > 1 + ? problem_.raman.n_components + : 0; + const std::array sizes{(stages + 1) * problem_.linear.n_phi_terms, + stages * problem_.linear.n_phi_terms, + stages * problem_.linear.n_prefactor_terms, stages, stages, stages, @@ -3770,7 +3820,7 @@ void GeneralWorkspace::allocate() { stages, stages * raman_terms, stages * raman_components, - problem_.n_phi_terms}; + problem_.linear.n_phi_terms}; std::size_t total = 0; for (std::size_t region = 0; region < sizes.size(); ++region) { stage_offsets_[region] = total; @@ -3781,7 +3831,8 @@ void GeneralWorkspace::allocate() { stage_scalars_ = RealView("solprop_native.step.scalars", 2 * total); bind_stage_scalars(0); // No trial's tapers yet: every scaling runs until one is bound. - trial_taper_.assign(problem_.n_stages, std::numeric_limits::quiet_NaN()); + trial_taper_.assign(problem_.stepper.n_stages, + std::numeric_limits::quiet_NaN()); // A pinned buffer's copy is asynchronous, so its source must not be rewritten // until the copy has run: `upload_trial` fences before the ring comes round again. // A host build's mirror is the buffer itself. @@ -3790,109 +3841,113 @@ void GeneralWorkspace::allocate() { stage_staging_.push_back( Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, stage_scalars_)); } - if (problem_.raman) { - const auto doubled = 2 * problem_.n_to; + if (problem_.raman.enabled) { + const auto doubled = 2 * problem_.dims.n_to; node_raman_drive_ = RealView2("solprop_native.raman.node_drive", - problem_.node_chunk * problem_.raman_drives(), doubled); - node_raman_spectrum_ = - ComplexView2("solprop_native.raman.node_spectrum", - problem_.node_chunk * problem_.raman_drives(), problem_.n_to + 1); + problem_.dims.node_chunk * problem_.raman_drives(), doubled); + node_raman_spectrum_ = ComplexView2( + "solprop_native.raman.node_spectrum", + problem_.dims.node_chunk * problem_.raman_drives(), problem_.dims.n_to + 1); node_raman_convolved_ = RealView2("solprop_native.raman.node_convolved", - problem_.node_chunk * problem_.raman_drives(), doubled); + problem_.dims.node_chunk * problem_.raman_drives(), doubled); raman_undamped_ = - RealView2("solprop_native.raman.undamped", problem_.n_raman_terms, doubled); + RealView2("solprop_native.raman.undamped", problem_.raman.n_terms, doubled); raman_kernel_time_ = RealView2("solprop_native.raman.kernel_time", problem_.raman_kernels(), doubled); - raman_kernel_spectrum_ = ComplexView2("solprop_native.raman.kernel_spectrum", - problem_.raman_kernels(), problem_.n_to + 1); + raman_kernel_spectrum_ = + ComplexView2("solprop_native.raman.kernel_spectrum", problem_.raman_kernels(), + problem_.dims.n_to + 1); raman_tensor_weights_ = - RealView2("solprop_native.raman.tensor_weights", problem_.n_raman_terms, 2); - cached_raman_rates_.resize(problem_.n_raman_terms); - if (problem_.n_raman_components > 1) { + RealView2("solprop_native.raman.tensor_weights", problem_.raman.n_terms, 2); + cached_raman_rates_.resize(problem_.raman.n_terms); + if (problem_.raman.n_components > 1) { raman_row_component_ = - RealView("solprop_native.raman.row_component", problem_.n_raman_terms); + RealView("solprop_native.raman.row_component", problem_.raman.n_terms); raman_component_fraction_ = RealView("solprop_native.raman.component_fraction", - problem_.n_raman_components); - cached_raman_densities_.resize(problem_.n_raman_components); + problem_.raman.n_components); + cached_raman_densities_.resize(problem_.raman.n_components); } } if (use_packed_radial_blas()) { - radial_blas_input_ = RealView2("solprop_native.radial.blas_input", problem_.n_nodes, - 2 * problem_.n_to); + radial_blas_input_ = RealView2("solprop_native.radial.blas_input", + problem_.dims.n_nodes, 2 * problem_.dims.n_to); radial_blas_output_ = RealView2("solprop_native.radial.blas_output", - problem_.n_nodes, 2 * problem_.n_to); + problem_.dims.n_nodes, 2 * problem_.dims.n_to); } // The hybrid planar kind's slab rule in x, (n_x, n_p) and (n_p, n_x). - const auto synthesis_rows = problem_.slab_geometry() ? problem_.n_x : transform_rows; + const auto synthesis_rows = + problem_.slab_geometry() ? problem_.spatial.n_x : transform_rows; const auto synthesis_columns = - problem_.slab_geometry() ? problem_.n_p : problem_.n_modes; + problem_.slab_geometry() ? problem_.spatial.n_p : problem_.dims.n_modes; synthesis_ = RealView2("solprop_native.transform.synthesis", synthesis_rows, synthesis_columns); projection_ = RealView2("solprop_native.transform.projection", synthesis_columns, synthesis_rows); scaled_synthesis_ = RealView2("solprop_native.transform.scaled_synthesis", - transform_rows, problem_.n_modes); + transform_rows, problem_.dims.n_modes); scaled_projection_ = RealView2("solprop_native.transform.scaled_projection", - problem_.n_modes, transform_rows); - if (problem_.family == FieldFamily::envelope && use_blas_for_synthesis()) { + problem_.dims.n_modes, transform_rows); + if (problem_.dims.family == FieldFamily::envelope && use_blas_for_synthesis()) { scaled_synthesis_complex_ = ComplexView2("solprop_native.transform.scaled_synthesis_complex", - transform_rows, problem_.n_modes); + transform_rows, problem_.dims.n_modes); scaled_projection_complex_ = ComplexView2("solprop_native.transform.scaled_projection_complex", - problem_.n_modes, transform_rows); + problem_.dims.n_modes, transform_rows); } - if (!problem_.projection_coarse_matrix.empty()) { + if (!problem_.transform.projection_coarse_matrix.empty()) { projection_coarse_ = RealView2("solprop_native.transform.projection_coarse", - problem_.n_modes, transform_rows); + problem_.dims.n_modes, transform_rows); scaled_projection_coarse_ = - RealView2("solprop_native.transform.scaled_projection_coarse", problem_.n_modes, - transform_rows); - if (problem_.family == FieldFamily::envelope && use_blas_for_projection()) { + RealView2("solprop_native.transform.scaled_projection_coarse", + problem_.dims.n_modes, transform_rows); + if (problem_.dims.family == FieldFamily::envelope && use_blas_for_projection()) { scaled_projection_coarse_complex_ = ComplexView2("solprop_native.transform.scaled_projection_coarse_complex", - problem_.n_modes, transform_rows); + problem_.dims.n_modes, transform_rows); scaled_projection_stacked_complex_ = ComplexView2("solprop_native.transform.scaled_projection_stacked_complex", - 2 * problem_.n_modes, transform_rows); + 2 * problem_.dims.n_modes, transform_rows); projected_stacked_complex_ = ComplexView2("solprop_native.response.projected_stacked_complex", 2 * columns, - problem_.n_to); - } else if (problem_.family != FieldFamily::envelope) { + problem_.dims.n_to); + } else if (problem_.dims.family != FieldFamily::envelope) { scaled_projection_stacked_ = RealView2("solprop_native.transform.scaled_projection_stacked", - 2 * problem_.n_modes, transform_rows); + 2 * problem_.dims.n_modes, transform_rows); projected_stacked_real_ = RealView2("solprop_native.response.projected_stacked_real", - 2 * temporal_columns, problem_.n_to); + 2 * temporal_columns, problem_.dims.n_to); } } - const auto chunk_transform_rows = problem_.node_chunk * problem_.n_pol_transform; - const auto tail_nodes = problem_.n_nodes % problem_.node_chunk; - const auto tail_transform_rows = tail_nodes * problem_.n_pol_transform; + const auto chunk_transform_rows = + problem_.dims.node_chunk * problem_.dims.n_pol_transform; + const auto tail_nodes = problem_.dims.n_nodes % problem_.dims.node_chunk; + const auto tail_transform_rows = tail_nodes * problem_.dims.n_pol_transform; if (use_chunked_blas_for_synthesis()) { - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { chunk_synthesis_complex_ = ComplexView2("solprop_native.transform.chunk_synthesis_complex", - chunk_transform_rows, problem_.n_modes); + chunk_transform_rows, problem_.dims.n_modes); if (tail_transform_rows != 0) tail_synthesis_complex_ = ComplexView2("solprop_native.transform.tail_synthesis_complex", - tail_transform_rows, problem_.n_modes); + tail_transform_rows, problem_.dims.n_modes); } else { chunk_synthesis_ = RealView2("solprop_native.transform.chunk_synthesis", - chunk_transform_rows, problem_.n_modes); + chunk_transform_rows, problem_.dims.n_modes); if (tail_transform_rows != 0) tail_synthesis_ = RealView2("solprop_native.transform.tail_synthesis", - tail_transform_rows, problem_.n_modes); + tail_transform_rows, problem_.dims.n_modes); } } if (use_chunked_blas_for_projection()) { - const bool stacked = !problem_.projection_coarse_matrix.empty(); - const auto projection_rows = stacked ? 2 * problem_.n_modes : problem_.n_modes; - if (problem_.family == FieldFamily::envelope) { + const bool stacked = !problem_.transform.projection_coarse_matrix.empty(); + const auto projection_rows = + stacked ? 2 * problem_.dims.n_modes : problem_.dims.n_modes; + if (problem_.dims.family == FieldFamily::envelope) { if (stacked) { chunk_projection_stacked_complex_ = ComplexView2("solprop_native.transform.chunk_projection_stacked_complex", @@ -3928,9 +3983,9 @@ void GeneralWorkspace::allocate() { } fine_spectrum_ = ComplexView2("solprop_native.fft.fine_spectrum", temporal_columns, fine_spectrum_size()); - if (!problem_.separable_geometry() && problem_.n_omega != fine_spectrum_size()) + if (!problem_.separable_geometry() && problem_.dims.n_omega != fine_spectrum_size()) fine_gaps_ = IndexView("solprop_native.fft.fine_gaps", 3 * fine_gap_runs().size()); - if (problem_.separable_geometry() && problem_.n_omega != fine_spectrum_size()) { + if (problem_.separable_geometry() && problem_.dims.n_omega != fine_spectrum_size()) { std::vector bins; for (const auto& [bin, length] : fine_gap_runs()) for (std::size_t offset = 0; offset < length; ++offset) @@ -3943,20 +3998,17 @@ void GeneralWorkspace::allocate() { // and one spectrum. const bool pack = packs_forward(); const std::size_t absorber_rows = - pack && problem_.in_stage_absorber ? temporal_columns : 0; + pack && problem_.absorber.in_stage ? temporal_columns : 0; const std::size_t coarse_rows = - pack && !problem_.projection_coarse_matrix.empty() ? temporal_columns : 0; + pack && !problem_.transform.projection_coarse_matrix.empty() ? temporal_columns + : 0; const std::size_t pack_rows = absorber_rows + temporal_columns + coarse_rows + (held_plasma() ? 2 * (temporal_columns + coarse_rows) : 0); const auto pack_slice = [&](const std::size_t first, const std::size_t count) { return std::pair(first, first + count); }; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr bool host_pack_layout = false; -#else - constexpr bool host_pack_layout = true; -#endif + constexpr bool host_pack_layout = build::host; // A host's pack: [absorber | fine | held fine x2 | coarse | held coarse x2]; a // device's: [absorber | fine | coarse | (held fine, held coarse) x2]. const std::size_t held_fine_rows = held_plasma() ? 2 * temporal_columns : 0; @@ -3966,7 +4018,7 @@ void GeneralWorkspace::allocate() { forward_pack_absorber_rows_ = absorber_rows; if (pack) { forward_pack_ = - RealView2("solprop_native.fft.forward_pack", pack_rows, problem_.n_to); + RealView2("solprop_native.fft.forward_pack", pack_rows, problem_.dims.n_to); forward_pack_spectrum_ = ComplexView2("solprop_native.fft.forward_pack_spectrum", pack_rows, projected_spectrum_size()); if (absorber_rows != 0 && pack_rows - absorber_rows > temporal_columns) { @@ -4005,7 +4057,7 @@ void GeneralWorkspace::allocate() { Kokkos::ALL)) : ComplexView2("solprop_native.fft.projected_spectrum", temporal_columns, projected_spectrum_size()); - if (!problem_.projection_coarse_matrix.empty()) + if (!problem_.transform.projection_coarse_matrix.empty()) projected_coarse_spectrum_ = coarse_rows != 0 ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, @@ -4019,80 +4071,82 @@ void GeneralWorkspace::allocate() { pack_slice(0, absorber_rows), Kokkos::ALL)) : ComplexView2("solprop_native.fft.absorber_spectrum", temporal_columns, projected_spectrum_size()); - if (problem_.family == FieldFamily::field_real) { + if (problem_.dims.family == FieldFamily::field_real) { modal_real_ = RealView2("solprop_native.fft.modal_time_real", temporal_columns, - problem_.n_to); + problem_.dims.n_to); node_real_ = separable_aliased_ ? modal_real_ : RealView2("solprop_native.response.node_real", - chunk_rows, problem_.n_to); + chunk_rows, problem_.dims.n_to); } else { modal_complex_ = ComplexView2("solprop_native.fft.modal_time_complex", - temporal_columns, problem_.n_to); + temporal_columns, problem_.dims.n_to); node_complex_ = separable_aliased_ ? modal_complex_ : ComplexView2("solprop_native.response.node_complex", - chunk_rows, problem_.n_to); + chunk_rows, problem_.dims.n_to); retained_synthesis_ = retained_synthesis(); if (retained_synthesis_) { retained_modal_ = ComplexView2("solprop_native.transform.retained_modal", - temporal_columns, problem_.n_omega); + temporal_columns, problem_.dims.n_omega); retained_node_ = ComplexView2("solprop_native.transform.retained_node", - chunk_rows, problem_.n_omega); + chunk_rows, problem_.dims.n_omega); } } - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { projected_complex_ = ComplexView2("solprop_native.response.projected_complex", - temporal_columns, problem_.n_to); + temporal_columns, problem_.dims.n_to); polarisation_complex_ = separable_aliased_ ? projected_complex_ : ComplexView2("solprop_native.response.polarisation_complex", chunk_rows, - problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) + problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) projected_coarse_complex_ = ComplexView2("solprop_native.response.projected_coarse_complex", - temporal_columns, problem_.n_to); + temporal_columns, problem_.dims.n_to); absorber_complex_ = ComplexView2("solprop_native.absorber.fine_complex", - temporal_columns, problem_.n_to); + temporal_columns, problem_.dims.n_to); } else { projected_real_ = pack ? RealView2(Kokkos::subview(forward_pack_, pack_slice(absorber_rows, temporal_columns), Kokkos::ALL)) : RealView2("solprop_native.response.projected_real", temporal_columns, - problem_.n_to); + problem_.dims.n_to); polarisation_real_ = separable_aliased_ ? projected_real_ : RealView2("solprop_native.response.polarisation_real", - chunk_rows, problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) + chunk_rows, problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) projected_coarse_real_ = coarse_rows != 0 ? RealView2(Kokkos::subview( forward_pack_, pack_slice(coarse_first, coarse_rows), Kokkos::ALL)) : RealView2("solprop_native.response.projected_coarse_real", - temporal_columns, problem_.n_to); + temporal_columns, problem_.dims.n_to); absorber_real_ = absorber_rows != 0 ? RealView2(Kokkos::subview(forward_pack_, pack_slice(0, absorber_rows), Kokkos::ALL)) : RealView2("solprop_native.absorber.fine_real", temporal_columns, - problem_.n_to); + problem_.dims.n_to); } if (field_plasma()) - held_current_ = RealView2("solprop_native.held.current", chunk_rows, problem_.n_to); + held_current_ = + RealView2("solprop_native.held.current", chunk_rows, problem_.dims.n_to); if (gated_plasma()) { - gated_prefix_ = RealView2("solprop_native.gated.prefix", chunk_rows, problem_.n_to); - if (problem_.plasma_band) + gated_prefix_ = + RealView2("solprop_native.gated.prefix", chunk_rows, problem_.dims.n_to); + if (problem_.plasma.band) gated_filtered_ = - RealView2("solprop_native.gated.filtered", chunk_rows, problem_.n_to); + RealView2("solprop_native.gated.filtered", chunk_rows, problem_.dims.n_to); } if (held_plasma()) { - held_derivative_ = RealView("solprop_native.held.derivative", problem_.n_to); - held_inverse_ = RealView("solprop_native.held.inverse", problem_.n_omega); + held_derivative_ = RealView("solprop_native.held.derivative", problem_.dims.n_to); + held_inverse_ = RealView("solprop_native.held.inverse", problem_.dims.n_omega); const bool radial = held_radial(); const auto rows = radial ? chunk_rows : temporal_columns; - const bool coarse = !problem_.projection_coarse_matrix.empty(); + const bool coarse = !problem_.transform.projection_coarse_matrix.empty(); // On a device the two currents' fine and coarse rows are one buffer, a stream's // fine rows then its coarse ones, so that one transform takes all four: on an AMD // Instinct MI325X each transform of a single-mode run was four library kernels of a @@ -4116,20 +4170,20 @@ void GeneralWorkspace::allocate() { Kokkos::subview(forward_pack_spectrum_, held_rows, Kokkos::ALL); } else if (packed) { const std::size_t held_rows = 2 * (coarse ? 2 * rows : rows); - held_pack_ = RealView2("solprop_native.held.pack", held_rows, problem_.n_to); + held_pack_ = RealView2("solprop_native.held.pack", held_rows, problem_.dims.n_to); held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", held_rows, - problem_.n_to / 2 + 1); + problem_.dims.n_to / 2 + 1); } else if (radial_packed) { - held_pack_ = RealView2("solprop_native.held.pack", 3 * rows, problem_.n_to); + held_pack_ = RealView2("solprop_native.held.pack", 3 * rows, problem_.dims.n_to); held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", 3 * rows, - problem_.n_to / 2 + 1); + problem_.dims.n_to / 2 + 1); } for (std::size_t index = 0; index < (radial ? 3U : 2U); ++index) { auto& stream = held_streams_[index]; const auto label = std::string("solprop_native.held.stream") + std::to_string(index) + "."; if (index < 2) - stream.node = RealView2(label + "node", chunk_rows, problem_.n_to); + stream.node = RealView2(label + "node", chunk_rows, problem_.dims.n_to); if (host_held && index < 2) { const auto fine = pack_slice(absorber_rows + rows + index * rows, rows); stream.projected = Kokkos::subview(forward_pack_, fine, Kokkos::ALL); @@ -4139,7 +4193,7 @@ void GeneralWorkspace::allocate() { stream.coarse = Kokkos::subview(forward_pack_, held_coarse, Kokkos::ALL); stream.coarse_spectrum = Kokkos::subview(forward_pack_spectrum_, held_coarse, Kokkos::ALL); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); } continue; } @@ -4158,7 +4212,7 @@ void GeneralWorkspace::allocate() { Kokkos::subview(held_pack_, rows_from(first + rows, rows), Kokkos::ALL); stream.coarse_spectrum = Kokkos::subview( held_pack_spectrum_, rows_from(first + rows, rows), Kokkos::ALL); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); } continue; } @@ -4168,109 +4222,113 @@ void GeneralWorkspace::allocate() { stream.projected = Kokkos::subview(held_pack_, third, Kokkos::ALL); stream.spectrum = Kokkos::subview(held_pack_spectrum_, third, Kokkos::ALL); } else { - stream.projected = RealView2(label + "projected", rows, problem_.n_to); - stream.spectrum = ComplexView2(label + "spectrum", rows, problem_.n_to / 2 + 1); + stream.projected = RealView2(label + "projected", rows, problem_.dims.n_to); + stream.spectrum = + ComplexView2(label + "spectrum", rows, problem_.dims.n_to / 2 + 1); } if (coarse) { - stream.coarse = RealView2(label + "coarse", rows, problem_.n_to); + stream.coarse = RealView2(label + "coarse", rows, problem_.dims.n_to); stream.coarse_spectrum = - ComplexView2(label + "coarse_spectrum", rows, problem_.n_to / 2 + 1); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); + ComplexView2(label + "coarse_spectrum", rows, problem_.dims.n_to / 2 + 1); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); } } if (radial) { held_nodal_spectrum_ = ComplexView2("solprop_native.held.nodal_spectrum", - chunk_rows, problem_.n_omega); - held_projected_spectrum_ = ComplexView2("solprop_native.held.projected_spectrum", - problem_.n_columns(), problem_.n_omega); - held_projection_ = RealView2("solprop_native.held.projection", problem_.n_modes, - problem_.node_chunk * problem_.n_pol_transform); + chunk_rows, problem_.dims.n_omega); + held_projected_spectrum_ = + ComplexView2("solprop_native.held.projected_spectrum", problem_.n_columns(), + problem_.dims.n_omega); + held_projection_ = + RealView2("solprop_native.held.projection", problem_.dims.n_modes, + problem_.dims.node_chunk * problem_.dims.n_pol_transform); } } absorber_fine_ = - RealView("solprop_native.absorber.rate_fine", problem_.absorber_fine.size()); + RealView("solprop_native.absorber.rate_fine", problem_.absorber.fine.size()); absorber_coarse_ = - RealView("solprop_native.absorber.rate_coarse", problem_.absorber_coarse.size()); - if (!problem_.spatial_absorber_matrix.empty()) { - if (problem_.family == FieldFamily::envelope) { + RealView("solprop_native.absorber.rate_coarse", problem_.absorber.coarse.size()); + if (!problem_.spatial.absorber_matrix.empty()) { + if (problem_.dims.family == FieldFamily::envelope) { spatial_absorber_complex_ = - ComplexView2("solprop_native.absorber.spatial_complex", problem_.n_modes, - problem_.n_modes); + ComplexView2("solprop_native.absorber.spatial_complex", problem_.dims.n_modes, + problem_.dims.n_modes); std::vector matrix; - matrix.reserve(problem_.spatial_absorber_matrix.size()); - for (const auto value : problem_.spatial_absorber_matrix) + matrix.reserve(problem_.spatial.absorber_matrix.size()); + for (const auto value : problem_.spatial.absorber_matrix) matrix.emplace_back(value, 0.0); copy_vector(execution_space_, spatial_absorber_complex_, matrix); } else { spatial_absorber_real_ = RealView2("solprop_native.absorber.spatial_real", - problem_.n_modes, problem_.n_modes); + problem_.dims.n_modes, problem_.dims.n_modes); copy_vector(execution_space_, spatial_absorber_real_, - problem_.spatial_absorber_matrix); + problem_.spatial.absorber_matrix); } } - if (problem_.envelope_plasma) { - envelope_rule_nodes_ = - RealView3("solprop_native.envelope.rule_nodes", problem_.n_channels, 4, 512); - envelope_rule_weights_ = - RealView3("solprop_native.envelope.rule_weights", problem_.n_channels, 4, 512); + if (problem_.plasma.envelope) { + envelope_rule_nodes_ = RealView3("solprop_native.envelope.rule_nodes", + problem_.plasma.n_channels, 4, 512); + envelope_rule_weights_ = RealView3("solprop_native.envelope.rule_weights", + problem_.plasma.n_channels, 4, 512); envelope_moments_ = RealView3("solprop_native.envelope.moments", 2, - problem_.node_chunk, problem_.n_to); - envelope_rates_ = - RealView2("solprop_native.envelope.rates", problem_.node_chunk, problem_.n_to); + problem_.dims.node_chunk, problem_.dims.n_to); + envelope_rates_ = RealView2("solprop_native.envelope.rates", + problem_.dims.node_chunk, problem_.dims.n_to); envelope_exposure_ = RealView2("solprop_native.envelope.exposure", - problem_.node_chunk, problem_.n_to); + problem_.dims.node_chunk, problem_.dims.n_to); envelope_failures_ = IndexView("solprop_native.envelope.failures", - problem_.node_chunk * problem_.n_to); + problem_.dims.node_chunk * problem_.dims.n_to); envelope_status_ = IndexView("solprop_native.envelope.status", 1); envelope_host_status_ = Kokkos::View( "solprop_native.envelope.host_status", 1); envelope_drude_factor_ = - RealView("solprop_native.envelope.drude_factor", problem_.n_omega); + RealView("solprop_native.envelope.drude_factor", problem_.dims.n_omega); envelope_loss_factor_ = - ComplexView("solprop_native.envelope.loss_factor", problem_.n_omega); + ComplexView("solprop_native.envelope.loss_factor", problem_.dims.n_omega); for (std::size_t index = 0; index < envelope_streams_.size(); ++index) { auto& stream = envelope_streams_[index]; const auto label = std::string("solprop_native.envelope.") + (index == 0 ? "drude." : "loss."); - stream.node = ComplexView2(label + "node", chunk_rows, problem_.n_to); + stream.node = ComplexView2(label + "node", chunk_rows, problem_.dims.n_to); stream.projected = - ComplexView2(label + "projected", temporal_columns, problem_.n_to); + ComplexView2(label + "projected", temporal_columns, problem_.dims.n_to); stream.spectrum = - ComplexView2(label + "spectrum", temporal_columns, problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) { - stream.coarse = ComplexView2(label + "coarse", temporal_columns, problem_.n_to); - stream.coarse_spectrum = - ComplexView2(label + "coarse_spectrum", temporal_columns, problem_.n_to); + ComplexView2(label + "spectrum", temporal_columns, problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) { + stream.coarse = + ComplexView2(label + "coarse", temporal_columns, problem_.dims.n_to); + stream.coarse_spectrum = ComplexView2(label + "coarse_spectrum", + temporal_columns, problem_.dims.n_to); } if (!projected_stacked_complex_.empty()) stream.stacked = - ComplexView2(label + "stacked", 2 * temporal_columns, problem_.n_to); + ComplexView2(label + "stacked", 2 * temporal_columns, problem_.dims.n_to); } } - if (problem_.plasma) { - if (problem_.n_channels > 1 || problem_.envelope_plasma) { - channel_parameters_ = - RealView2("solprop_native.plasma.channel_parameters", problem_.n_channels, 9); + if (problem_.plasma.enabled) { + if (problem_.plasma.n_channels > 1 || problem_.plasma.envelope) { + channel_parameters_ = RealView2("solprop_native.plasma.channel_parameters", + problem_.plasma.n_channels, 9); channel_coefficients_ = RealView("solprop_native.plasma.channel_coefficients", - problem_.channel_coefficients.size()); - if (!problem_.envelope_plasma) - channel_scratch_ = - decltype(channel_scratch_)("solprop_native.plasma.channel_scratch", - problem_.n_channels, problem_.node_chunk, 4); + problem_.plasma.channel_coefficients.size()); + if (!problem_.plasma.envelope) + channel_scratch_ = decltype(channel_scratch_)( + "solprop_native.plasma.channel_scratch", problem_.plasma.n_channels, + problem_.dims.node_chunk, 4); } rate_coefficients_ = RealView("solprop_native.plasma.rate_coefficients", - problem_.rate_coefficients.size()); + problem_.plasma.rate_coefficients.size()); plasma_statistics_scratch_ = - RealView2("solprop_native.plasma.statistics_scratch", problem_.n_nodes, - 1 + 2 * problem_.n_channels); - if (!problem_.envelope_plasma) { + RealView2("solprop_native.plasma.statistics_scratch", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); + if (!problem_.plasma.envelope) { // solprop SEP 18: the rule's taps and the run diagnostics' six parts per node. plasma_taps_ = - RealView("solprop_native.plasma.taps", problem_.plasma_taps.size()); + RealView("solprop_native.plasma.taps", problem_.plasma.taps.size()); plasma_diagnostics_ = RealView2("solprop_native.plasma.diagnostics", - problem_.n_nodes, plasma_diagnostic_parts); + problem_.dims.n_nodes, plasma_diagnostic_parts); } - if (!problem_.envelope_plasma) { + if (!problem_.plasma.envelope) { // The band rule's team column and its statistics (`team_band_plasma_column`). // A device keeps the trapezoid device column's rows, one per node of a chunk, // and the band column reuses them. A host team owns one row of each per slot, @@ -4281,25 +4339,25 @@ void GeneralWorkspace::allocate() { // tiled column (solprop SEP 25 section 6.7.6) keeps the device's rows on every // build, the weight rows holding the free-current local prefixes, and the tiles' // totals by row. - const bool tiled = band_tiled_ && problem_.plasma_band; + const bool tiled = band_tiled_ && problem_.plasma.band; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - band_slots_ = problem_.node_chunk; + band_slots_ = problem_.dims.node_chunk; const auto guard_rows = chunk_rows; const auto weight_rows = chunk_rows; const auto component_rows = chunk_rows; #else if (tiled) { - band_slots_ = problem_.node_chunk; + band_slots_ = problem_.dims.node_chunk; } else { - band_slots_ = std::min(problem_.node_chunk, + band_slots_ = std::min(problem_.dims.node_chunk, static_cast(std::max( 1, response_execution_space_.concurrency()))); } - const bool band = problem_.plasma_band; + const bool band = problem_.plasma.band; const auto guard_rows = tiled ? chunk_rows : band ? band_slots_ : 0; - const auto weight_rows = tiled ? chunk_rows - : band && problem_.n_channels > 1 ? band_slots_ - : 0; + const auto weight_rows = tiled ? chunk_rows + : band && problem_.plasma.n_channels > 1 ? band_slots_ + : 0; const auto component_rows = band ? checked_product(band_slots_, problem_.effective_node_pol(), "band component rows") @@ -4316,8 +4374,8 @@ void GeneralWorkspace::allocate() { largest /= 2; if (band_tile_bytes(largest) > scratch_limit) throw std::length_error("the band tile exceeds the team scratch"); - band_tile_ = band_tile_samples(problem_.n_to, largest); - band_tile_count_ = band_tile_count(problem_.n_to, band_tile_); + band_tile_ = band_tile_samples(problem_.dims.n_to, largest); + band_tile_count_ = band_tile_count(problem_.dims.n_to, band_tile_); require_allocation(checked_product(chunk_rows, band_tile_count_, "band tiles"), sizeof(double), "band-tile bytes"); band_tile_totals_ = RealView2("solprop_native.plasma.band_tile_totals", @@ -4327,44 +4385,47 @@ void GeneralWorkspace::allocate() { checked_sum(checked_product(3, band_slots_, "band rows"), guard_rows, "band rows"), checked_sum(weight_rows, component_rows, "band rows"), "band rows"); - require_allocation(checked_product(band_rows, problem_.n_to, "band rows"), + require_allocation(checked_product(band_rows, problem_.dims.n_to, "band rows"), sizeof(double), "band-row bytes"); plasma_rate_ = - RealView2("solprop_native.plasma.rate", band_slots_, problem_.n_to); + RealView2("solprop_native.plasma.rate", band_slots_, problem_.dims.n_to); plasma_exposure_ = - RealView2("solprop_native.plasma.exposure", band_slots_, problem_.n_to); + RealView2("solprop_native.plasma.exposure", band_slots_, problem_.dims.n_to); plasma_fraction_ = - RealView2("solprop_native.plasma.fraction", band_slots_, problem_.n_to); + RealView2("solprop_native.plasma.fraction", band_slots_, problem_.dims.n_to); if (component_rows != 0) { plasma_current_integrand_ = RealView2("solprop_native.plasma.current_integrand", - component_rows, problem_.n_to); + component_rows, problem_.dims.n_to); } if (guard_rows != 0) - plasma_free_current_ = - RealView2("solprop_native.plasma.free_current", guard_rows, problem_.n_to); + plasma_free_current_ = RealView2("solprop_native.plasma.free_current", + guard_rows, problem_.dims.n_to); if (weight_rows != 0) plasma_outer_current_ = RealView2("solprop_native.plasma.outer_current", - weight_rows, problem_.n_to); + weight_rows, problem_.dims.n_to); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - if (!problem_.plasma_band) { - plasma_tiles_ = (problem_.n_to + plasma_tile_samples - 1) / plasma_tile_samples; + if (!problem_.plasma.band) { + plasma_tiles_ = + (problem_.dims.n_to + plasma_tile_samples - 1) / plasma_tile_samples; plasma_tile_totals_ = RealView2("solprop_native.plasma.tile_totals", chunk_rows, plasma_tiles_); } #endif } - if (problem_.stash_plasma) { - stash_candidate_ = RealView2("solprop_native.plasma.stash_candidate", - problem_.n_nodes, 1 + 2 * problem_.n_channels); - stash_current_ = RealView2("solprop_native.plasma.stash_current", - problem_.n_nodes, 1 + 2 * problem_.n_channels); + if (problem_.plasma.stash) { + stash_candidate_ = + RealView2("solprop_native.plasma.stash_candidate", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); + stash_current_ = + RealView2("solprop_native.plasma.stash_current", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); } } - if (problem_.split_step_absorber) { + if (problem_.absorber.split_step) { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - split_factor_ = RealView("solprop_native.absorber.split_factor", problem_.n_t); + split_factor_ = RealView("solprop_native.absorber.split_factor", problem_.dims.n_t); #endif - if (problem_.free_geometry() && problem_.family == FieldFamily::envelope) { + if (problem_.free_geometry() && problem_.dims.family == FieldFamily::envelope) { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // In place on the state's rows, a block holds no buffer of its own, only its // plans' work area (9 MiB on cuFFT for every row at 768 x 768 x 600), so a device @@ -4378,112 +4439,129 @@ void GeneralWorkspace::allocate() { } else { coarse_spectrum_ = ComplexView2("solprop_native.absorber.coarse_spectrum", temporal_columns, n); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) coarse_time_complex_ = ComplexView2("solprop_native.absorber.coarse_time_complex", - temporal_columns, problem_.n_t); + temporal_columns, problem_.dims.n_t); else coarse_time_real_ = RealView2("solprop_native.absorber.coarse_time_real", - temporal_columns, problem_.n_t); + temporal_columns, problem_.dims.n_t); } } - if (problem_.general_modal) { + if (problem_.general.enabled) { // solprop SEP 47: a general modal basis's statistics whiten the columns by the // metric's root (`whitened_statistics`), into scratch held from the start. statistics_whitened_ = ComplexView2("solprop_native.statistics.whitened", - problem_.n_columns(), problem_.n_omega); - statistics_root_ = - ComplexView("solprop_native.statistics.metric_root", - checked_product(problem_.n_omega, - checked_product(problem_.n_modes, problem_.n_modes, - "metric root"), - "metric root")); + problem_.n_columns(), problem_.dims.n_omega); + statistics_root_ = ComplexView( + "solprop_native.statistics.metric_root", + checked_product(problem_.dims.n_omega, + checked_product(problem_.dims.n_modes, problem_.dims.n_modes, + "metric root"), + "metric root")); statistics_whitened_sums_ = RealView("solprop_native.statistics.whitened_sums", - problem_.n_columns() + problem_.n_omega); + problem_.n_columns() + problem_.dims.n_omega); general_ = std::make_unique(problem_); // The source owns execution-space copies; the host maps are the largest arrays a // general guide carries and nothing reads them again. - std::vector().swap(problem_.general_synthesis); - std::vector().swap(problem_.general_projection); - std::vector().swap(problem_.general_tensor); - std::vector().swap(problem_.general_frequency_factor); - std::vector().swap(problem_.general_delayed_tensor); - std::vector().swap(problem_.general_delayed_spectrum); - std::vector().swap(problem_.general_node_response); - } - if (problem_.free_geometry() && !problem_.spatial_spectral_weights.empty()) + std::vector().swap(problem_.general.synthesis); + std::vector().swap(problem_.general.projection); + std::vector().swap(problem_.general.tensor); + std::vector().swap(problem_.general.frequency_factor); + std::vector().swap(problem_.general.delayed_tensor); + std::vector().swap(problem_.general.delayed_spectrum); + std::vector().swap(problem_.general.node_response); + } + if (problem_.free_geometry() && !problem_.spatial.spectral_weights.empty()) far_field_ = std::make_unique(problem_, execution_space_); - if (problem_.dispersive && problem_.separable_geometry()) { + if (problem_.dispersive.enabled && problem_.separable_geometry()) { // The planar kind's slab rule in x; free space has none. const bool slab = problem_.slab_geometry(); separable_dispersive_ = std::make_unique( problem_, kernel_pool_, - SeparableDispersive::StageViews{ - fine_spectrum_, projected_spectrum_, node_real_, polarisation_real_, - node_complex_, polarisation_complex_, slab ? synthesis_ : RealView2{}, - slab ? projection_ : RealView2{}, time_window_, carrier_}); - } else if (problem_.dispersive && !problem_.general_modal) { + SeparableDispersive::StageViews{.fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .synthesis = slab ? synthesis_ : RealView2{}, + .projection = slab ? projection_ : RealView2{}, + .time_window = time_window_, + .carrier = carrier_}); + } else if (problem_.dispersive.enabled && !problem_.general.enabled) { dispersive_ = std::make_unique( problem_, kernel_pool_, DispersiveResponse::StageViews{ - fine_spectrum_, projected_spectrum_, projected_coarse_spectrum_, node_real_, - polarisation_real_, node_complex_, polarisation_complex_, scaled_synthesis_, - scaled_projection_, scaled_projection_coarse_, time_window_, carrier_}); - } - if (problem_.dispersive) { + .fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .projected_coarse_spectrum = projected_coarse_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .scaled_synthesis = scaled_synthesis_, + .scaled_projection = scaled_projection_, + .scaled_projection_coarse = scaled_projection_coarse_, + .time_window = time_window_, + .carrier = carrier_}); + } + if (problem_.dispersive.enabled) { // The response holds execution-space copies of its layout (the general route's // source built its own above). - std::vector().swap(problem_.dispersive_inputs); - std::vector().swap(problem_.dispersive_term_multipliers); - std::vector().swap(problem_.dispersive_quartics); + std::vector().swap(problem_.dispersive.inputs); + std::vector().swap(problem_.dispersive.term_multipliers); + std::vector().swap(problem_.dispersive.quartics); } - if (problem_.coupled_flow) { + if (problem_.flow.enabled) { flow_ = std::make_unique(problem_, flow_execution_space_); - std::vector().swap(problem_.flow_frozen); - std::vector().swap(problem_.flow_residual_terms); - std::vector().swap(problem_.flow_left); - std::vector().swap(problem_.flow_right); - std::vector().swap(problem_.flow_eigenvalues); - std::vector().swap(problem_.flow_route); - std::vector().swap(problem_.flow_fallback_vectors); - std::vector().swap(problem_.flow_fallback_triangular); + std::vector().swap(problem_.flow.frozen); + std::vector().swap(problem_.flow.residual_terms); + std::vector().swap(problem_.flow.left); + std::vector().swap(problem_.flow.right); + std::vector().swap(problem_.flow.eigenvalues); + std::vector().swap(problem_.flow.route); + std::vector().swap(problem_.flow.fallback_vectors); + std::vector().swap(problem_.flow.fallback_triangular); } } void GeneralWorkspace::upload() { - if (problem_.n_raman_components > 1) { - copy_vector(execution_space_, raman_row_component_, problem_.raman_row_component); + if (problem_.raman.n_components > 1) { + copy_vector(execution_space_, raman_row_component_, problem_.raman.row_component); copy_vector(execution_space_, raman_component_fraction_, - problem_.raman_component_fraction); + problem_.raman.component_fraction); } if (problem_.separable_geometry()) { - copy_vector(execution_space_, spatial_.square, problem_.spatial_k_squared); - copy_vector(execution_space_, spatial_.square_low, problem_.spatial_k_squared_low); - copy_vector(execution_space_, spatial_.numerator, problem_.spatial_numerator); - copy_vector(execution_space_, spatial_.clamped, problem_.spatial_clamped); - copy_vector(execution_space_, spatial_.transverse, problem_.spatial_kperp_sq); + copy_vector(execution_space_, spatial_.square, problem_.spatial.k_squared); + copy_vector(execution_space_, spatial_.square_low, problem_.spatial.k_squared_low); + copy_vector(execution_space_, spatial_.numerator, problem_.spatial.numerator); + copy_vector(execution_space_, spatial_.clamped, problem_.spatial.clamped); + copy_vector(execution_space_, spatial_.transverse, problem_.spatial.kperp_sq); copy_vector(execution_space_, spatial_.transverse_low, - problem_.spatial_kperp_sq_low); - copy_vector(execution_space_, spatial_.frame, problem_.spatial_frame); + problem_.spatial.kperp_sq_low); + copy_vector(execution_space_, spatial_.frame, problem_.spatial.frame); copy_vector(execution_space_, spatial_.spectral_rate, - problem_.spatial_spectral_rate); - copy_vector(execution_space_, spatial_.user_rate, problem_.spatial_user_rate); - copy_vector(execution_space_, spatial_.absorber, problem_.spatial_absorber_rate); + problem_.spatial.spectral_rate); + copy_vector(execution_space_, spatial_.user_rate, problem_.spatial.user_rate); + copy_vector(execution_space_, spatial_.absorber, problem_.spatial.absorber_rate); copy_vector(execution_space_, spatial_.transverse_rate, - problem_.spatial_kperp_rate); + problem_.spatial.kperp_rate); if (problem_.slab_geometry()) - copy_vector(execution_space_, spatial_.kappa, problem_.spatial_kappa_sq); - } else if (!problem_.flux_k_squared.empty()) { - copy_vector(execution_space_, flux_.square, problem_.flux_k_squared); - copy_vector(execution_space_, flux_.square_low, problem_.flux_k_squared_low); - copy_vector(execution_space_, flux_.transverse, problem_.flux_kperp_sq); - copy_vector(execution_space_, flux_.transverse_low, problem_.flux_kperp_sq_low); - } - if (problem_.n_channels > 1 || problem_.envelope_plasma) { - copy_vector(execution_space_, channel_parameters_, problem_.channel_parameters); - copy_vector(execution_space_, channel_coefficients_, problem_.channel_coefficients); - } - copy_vector(execution_space_, crop_index_, problem_.crop_index); + copy_vector(execution_space_, spatial_.kappa, problem_.spatial.kappa_sq); + } else if (!problem_.flux.k_squared.empty()) { + copy_vector(execution_space_, flux_.square, problem_.flux.k_squared); + copy_vector(execution_space_, flux_.square_low, problem_.flux.k_squared_low); + copy_vector(execution_space_, flux_.transverse, problem_.flux.kperp_sq); + copy_vector(execution_space_, flux_.transverse_low, problem_.flux.kperp_sq_low); + } + if (problem_.plasma.n_channels > 1 || problem_.plasma.envelope) { + copy_vector(execution_space_, channel_parameters_, + problem_.plasma.channel_parameters); + copy_vector(execution_space_, channel_coefficients_, + problem_.plasma.channel_coefficients); + } + copy_vector(execution_space_, crop_index_, problem_.grid.crop_index); if (!fine_gaps_.empty()) { std::vector runs; std::size_t position = 0; @@ -4493,59 +4571,62 @@ void GeneralWorkspace::upload() { } copy_vector(execution_space_, fine_gaps_, runs); } - if (problem_.envelope_plasma) { - copy_vector(execution_space_, envelope_rule_nodes_, problem_.envelope_rule_nodes); + if (problem_.plasma.envelope) { + copy_vector(execution_space_, envelope_rule_nodes_, + problem_.plasma.envelope_rule_nodes); copy_vector(execution_space_, envelope_rule_weights_, - problem_.envelope_rule_weights); + problem_.plasma.envelope_rule_weights); copy_vector(execution_space_, envelope_drude_factor_, - problem_.envelope_drude_factor); - copy_vector(execution_space_, envelope_loss_factor_, problem_.envelope_loss_factor); - } - copy_vector(execution_space_, active_mask_, problem_.active_mask); - copy_vector(execution_space_, norm_mask_, problem_.norm_mask); - copy_vector(execution_space_, omega_window_, problem_.omega_window); - copy_vector(execution_space_, omega_, problem_.omega); - copy_vector(execution_space_, time_window_, problem_.time_window); + problem_.plasma.envelope_drude_factor); + copy_vector(execution_space_, envelope_loss_factor_, + problem_.plasma.envelope_loss_factor); + } + copy_vector(execution_space_, active_mask_, problem_.grid.active_mask); + copy_vector(execution_space_, norm_mask_, problem_.grid.norm_mask); + copy_vector(execution_space_, omega_window_, problem_.grid.omega_window); + copy_vector(execution_space_, omega_, problem_.grid.omega); + copy_vector(execution_space_, time_window_, problem_.grid.time_window); if (held_plasma()) { - copy_vector(execution_space_, held_derivative_, problem_.time_window_derivative); - std::vector inverse(problem_.n_omega, 0.0); - for (std::size_t index = 0; index < problem_.n_omega; ++index) - if (problem_.active_mask[index]) - inverse[index] = 1.0 / problem_.omega[index]; + copy_vector(execution_space_, held_derivative_, + problem_.grid.time_window_derivative); + std::vector inverse(problem_.dims.n_omega, 0.0); + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) + if (problem_.grid.active_mask[index] != 0) + inverse[index] = 1.0 / problem_.grid.omega[index]; copy_vector(execution_space_, held_inverse_, inverse); } - copy_vector(execution_space_, carrier_, problem_.carrier); + copy_vector(execution_space_, carrier_, problem_.grid.carrier); // A z table's slots start at zero (Kokkos initialises a view) until evaluated. lossless_phase_ = false; - if (!problem_.phi_table) { - copy_vector(execution_space_, phi_terms_, problem_.phi_terms); + if (!problem_.linear.phi_table) { + copy_vector(execution_space_, phi_terms_, problem_.linear.phi_terms); lossless_phase_ = - std::all_of(problem_.phi_terms.begin(), problem_.phi_terms.end(), - [](const Complex& term) { return term.real() == 0.0; }); + std::ranges::all_of(problem_.linear.phi_terms, + [](const Complex& term) { return term.real() == 0.0; }); } - if (!problem_.prefactor_table) - copy_vector(execution_space_, prefactor_terms_, problem_.prefactor_terms); - copy_vector(execution_space_, tableau_, problem_.tableau_matrix); + if (!problem_.linear.prefactor_table) + copy_vector(execution_space_, prefactor_terms_, problem_.linear.prefactor_terms); + copy_vector(execution_space_, tableau_, problem_.stepper.tableau_matrix); copy_vector(execution_space_, spatial_spectral_weights_, - problem_.spatial_spectral_weights); - copy_vector(execution_space_, spatial_on_axis_, problem_.spatial_on_axis); - copy_vector(execution_space_, synthesis_, problem_.synthesis_matrix); - copy_vector(execution_space_, projection_, problem_.projection_matrix); - if (!problem_.projection_coarse_matrix.empty()) + problem_.spatial.spectral_weights); + copy_vector(execution_space_, spatial_on_axis_, problem_.spatial.on_axis); + copy_vector(execution_space_, synthesis_, problem_.transform.synthesis_matrix); + copy_vector(execution_space_, projection_, problem_.transform.projection_matrix); + if (!problem_.transform.projection_coarse_matrix.empty()) copy_vector(execution_space_, projection_coarse_, - problem_.projection_coarse_matrix); - copy_vector(execution_space_, absorber_fine_, problem_.absorber_fine); - spatial_only_absorber_ = - std::all_of(problem_.absorber_fine.begin(), problem_.absorber_fine.end(), - [](const double rate) { return rate == 0.0; }); - copy_vector(execution_space_, absorber_coarse_, problem_.absorber_coarse); - if (problem_.plasma) - copy_vector(execution_space_, rate_coefficients_, problem_.rate_coefficients); + problem_.transform.projection_coarse_matrix); + copy_vector(execution_space_, absorber_fine_, problem_.absorber.fine); + spatial_only_absorber_ = std::ranges::all_of( + problem_.absorber.fine, [](const double rate) { return rate == 0.0; }); + copy_vector(execution_space_, absorber_coarse_, problem_.absorber.coarse); + if (problem_.plasma.enabled) + copy_vector(execution_space_, rate_coefficients_, + problem_.plasma.rate_coefficients); if (!plasma_taps_.empty()) - copy_vector(execution_space_, plasma_taps_, problem_.plasma_taps); - if (problem_.raman) { - copy_vector(execution_space_, raman_undamped_, problem_.raman_undamped); - copy_vector(execution_space_, raman_tensor_weights_, problem_.raman_tensor_weights); + copy_vector(execution_space_, plasma_taps_, problem_.plasma.taps); + if (problem_.raman.enabled) { + copy_vector(execution_space_, raman_undamped_, problem_.raman.undamped); + copy_vector(execution_space_, raman_tensor_weights_, problem_.raman.tensor_weights); graph::deep_copy(execution_space_, node_raman_drive_, 0.0); } } @@ -4556,6 +4637,7 @@ void GeneralWorkspace::upload() { // transform/propagation rounding budgets rather than bitwise agreement. const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, const std::size_t length) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide return fft_pool_.get(CpuOperation::fft, samples, length); } @@ -4585,7 +4667,7 @@ void GeneralWorkspace::seed_free_wisdom() { // slower codelets for long rows (a 16384-sample envelope GNLSE ran its pair at about // twice the measured cost). The plan policy (`fftw_seed_effort`) decides the effort. void GeneralWorkspace::seed_modal_wisdom() { - if (problem_.family == FieldFamily::field_real) { + if (problem_.dims.family == FieldFamily::field_real) { seed_row_fft_wisdom(fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, KokkosFFT::Direction::backward); graph::deep_copy(execution_space_, modal_real_, 0.0); @@ -4594,7 +4676,7 @@ void GeneralWorkspace::seed_modal_wisdom() { modal_complex_, KokkosFFT::Direction::backward); graph::deep_copy(execution_space_, modal_complex_, Complex{}); } - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { seed_row_fft_wisdom(fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, projected_complex_, Complex{}); @@ -4642,7 +4724,7 @@ void GeneralWorkspace::seed_stream_wisdom() { graph::deep_copy(execution_space_, stream.coarse, 0.0); graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); } - if (problem_.envelope_plasma) + if (problem_.plasma.envelope) for (auto& stream : envelope_streams_) { seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, KokkosFFT::Direction::forward); @@ -4671,30 +4753,32 @@ void GeneralWorkspace::plan() { fft_execution_space_ = modal_complex_.size() > modal_real_.size() ? fft_space_for_rows(modal_complex_) : fft_space_for_rows(modal_real_); - if (problem_.raman) { + if (problem_.raman.enabled) { raman_execution_space_ = raman_pool_.get( CpuOperation::raman, node_raman_drive_.size(), row_length(node_raman_drive_)); - const auto raman_tail = problem_.n_nodes % problem_.node_chunk; + const auto raman_tail = problem_.dims.n_nodes % problem_.dims.node_chunk; raman_tail_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_tail * problem_.raman_drives() * 2 * problem_.n_to, - 2 * problem_.n_to); + CpuOperation::raman, + raman_tail * problem_.raman_drives() * 2 * problem_.dims.n_to, + 2 * problem_.dims.n_to); raman_kernel_execution_space_ = raman_pool_.get( CpuOperation::raman, raman_kernel_time_.size(), row_length(raman_kernel_time_)); raman_multiply_space_ = kernel_pool_.get(CpuOperation::stage, node_raman_spectrum_.size()); raman_tail_multiply_space_ = - kernel_pool_.get(CpuOperation::stage, - raman_tail * problem_.raman_drives() * (problem_.n_to + 1)); + kernel_pool_.get(CpuOperation::stage, raman_tail * problem_.raman_drives() * + (problem_.dims.n_to + 1)); raman_rebuild_space_ = kernel_pool_.get( - CpuOperation::response, raman_kernel_time_.size() * problem_.n_raman_terms); + CpuOperation::response, raman_kernel_time_.size() * problem_.raman.n_terms); raman_scale_space_ = kernel_pool_.get(CpuOperation::stage, raman_kernel_spectrum_.size()); raman_products_ = raman_modal_products(); if (raman_products_ != 0) { raman_product_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_products_ * 2 * problem_.n_to, 2 * problem_.n_to); - raman_product_space_ = - kernel_pool_.get(CpuOperation::stage, raman_products_ * 2 * problem_.n_to); + CpuOperation::raman, raman_products_ * 2 * problem_.dims.n_to, + 2 * problem_.dims.n_to); + raman_product_space_ = kernel_pool_.get(CpuOperation::stage, + raman_products_ * 2 * problem_.dims.n_to); } } if (general_) @@ -4725,25 +4809,27 @@ void GeneralWorkspace::plan() { fft_space_for_rows(transverse_), transverse_, transverse_, KokkosFFT::Direction::backward, axes); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - bool identity_crop = problem_.crop_index.size() == problem_.n_omega; - for (std::size_t index = 0; identity_crop && index < problem_.n_omega; ++index) - identity_crop = problem_.crop_index[index] == index; - volume_kerr_ = identity_crop && problem_.family == FieldFamily::envelope && - problem_.n_pol_state == 1 && problem_.effective_node_pol() == 1 && - problem_.n_to == problem_.n_omega && problem_.n_t == problem_.n_to && - separable_aliased_ && problem_.nonlinear && !problem_.plasma && - !problem_.raman && !problem_.envelope_plasma && - !problem_.dispersive && separable_dispersive_ == nullptr && - problem_.harmonic_factor == 0.0 && separable_gaps_.empty(); + bool identity_crop = problem_.grid.crop_index.size() == problem_.dims.n_omega; + for (std::size_t index = 0; identity_crop && index < problem_.dims.n_omega; ++index) + identity_crop = problem_.grid.crop_index[index] == index; + volume_kerr_ = identity_crop && problem_.dims.family == FieldFamily::envelope && + problem_.dims.n_pol_state == 1 && + problem_.effective_node_pol() == 1 && + problem_.dims.n_to == problem_.dims.n_omega && + problem_.dims.n_t == problem_.dims.n_to && separable_aliased_ && + problem_.kerr.nonlinear && !problem_.plasma.enabled && + !problem_.raman.enabled && !problem_.plasma.envelope && + !problem_.dispersive.enabled && separable_dispersive_ == nullptr && + problem_.kerr.harmonic_factor == 0.0 && separable_gaps_.empty(); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // At 768 x 768 x 600 on an NVIDIA H200, cuFFT's `{N,N,T}` takes 14.7 ms a transform // against 17.6 ms for `{T,N,N}` (solprop #365). An in-stage absorber evaluates its // term from the same input in the scratch's order, so it keeps that order. - volume_order_ = volume_kerr_ && !problem_.in_stage_absorber; + volume_order_ = volume_kerr_ && !problem_.absorber.in_stage; #endif if (volume_order_) - volume_scratch_ = ComplexView3(transverse_.data(), problem_.n_y, problem_.n_x, - problem_.n_omega); + volume_scratch_ = ComplexView3(transverse_.data(), problem_.spatial.n_y, + problem_.spatial.n_x, problem_.dims.n_omega); if (volume_kerr_) { constexpr std::size_t volume_axes = 3; const auto& volume = volume_order_ ? volume_scratch_ : transverse_; @@ -4790,7 +4876,7 @@ void GeneralWorkspace::plan() { fft_space_for_rows(held_pack_), held_pack_, held_pack_spectrum_, KokkosFFT::Direction::forward, axis); } - if (problem_.envelope_plasma) + if (problem_.plasma.envelope) for (auto& stream : envelope_streams_) { stream.forward = std::make_unique( fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, @@ -4803,7 +4889,7 @@ void GeneralWorkspace::plan() { statistics_inverse_plan_ = std::make_unique( fft_space_for_rows(statistics_time_), statistics_spectrum_, statistics_time_, KokkosFFT::Direction::backward, axis); - if (problem_.family == FieldFamily::field_real) { + if (problem_.dims.family == FieldFamily::field_real) { inverse_real_plan_ = std::make_unique( fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, KokkosFFT::Direction::backward, axis); @@ -4829,18 +4915,18 @@ void GeneralWorkspace::plan() { graph::deep_copy(execution_space_, node, Complex{}); return plan; }; - retained_inverse_plan_ = plan_rows(problem_.node_chunk); - if (const auto tail = problem_.n_nodes % problem_.node_chunk; tail != 0) + retained_inverse_plan_ = plan_rows(problem_.dims.node_chunk); + if (const auto tail = problem_.dims.n_nodes % problem_.dims.node_chunk; tail != 0) retained_tail_inverse_plan_ = plan_rows(tail); } - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { forward_complex_plan_ = std::make_unique( fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, KokkosFFT::Direction::forward, axis); absorber_complex_plan_ = std::make_unique( fft_space_for_rows(absorber_complex_), absorber_complex_, absorber_spectrum_, KokkosFFT::Direction::forward, axis); - if (!problem_.projection_coarse_matrix.empty()) + if (!problem_.transform.projection_coarse_matrix.empty()) forward_coarse_complex_plan_ = std::make_unique( fft_space_for_rows(projected_coarse_complex_), projected_coarse_complex_, projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); @@ -4871,7 +4957,7 @@ void GeneralWorkspace::plan() { forward_pack_coarse_plan_ = std::make_unique( fft_space_for_rows(forward_pack_coarse_), forward_pack_coarse_, forward_pack_coarse_spectrum_, KokkosFFT::Direction::forward, axis); - if (!problem_.projection_coarse_matrix.empty()) + if (!problem_.transform.projection_coarse_matrix.empty()) forward_coarse_real_plan_ = std::make_unique( fft_space_for_rows(projected_coarse_real_), projected_coarse_real_, projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); @@ -4881,13 +4967,13 @@ void GeneralWorkspace::plan() { const auto full_units = transverse_units_per_chunk(); const auto tail_units = transverse_units() % full_units; const auto rows_per_unit = - problem_.slab_geometry() ? problem_.n_p : problem_.n_pol_state; + problem_.slab_geometry() ? problem_.spatial.n_p : problem_.dims.n_pol_state; temporal_tail_rows_ = tail_units * rows_per_unit; if (temporal_tail_rows_ != 0) { const auto& space = - fft_space_for(temporal_tail_rows_ * problem_.n_to, problem_.n_to); + fft_space_for(temporal_tail_rows_ * problem_.dims.n_to, problem_.dims.n_to); const auto rows = fine_spectrum_.extent(0); - if (problem_.family == FieldFamily::field_real) + if (problem_.dims.family == FieldFamily::field_real) temporal_tail_inverse_real_.create(space, fine_spectrum_, modal_real_, rows, temporal_tail_rows_, KokkosFFT::Direction::backward); @@ -4895,7 +4981,7 @@ void GeneralWorkspace::plan() { temporal_tail_inverse_complex_.create(space, fine_spectrum_, modal_complex_, rows, temporal_tail_rows_, KokkosFFT::Direction::backward); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) temporal_tail_forward_complex_.create( space, projected_complex_, projected_spectrum_, rows, temporal_tail_rows_, KokkosFFT::Direction::forward); @@ -4906,18 +4992,18 @@ void GeneralWorkspace::plan() { // The tail plans' measured planning overwrote their rows. graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); graph::deep_copy(execution_space_, projected_spectrum_, Complex{}); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) graph::deep_copy(execution_space_, projected_complex_, Complex{}); else graph::deep_copy(execution_space_, projected_real_, 0.0); - if (problem_.family == FieldFamily::field_real) + if (problem_.dims.family == FieldFamily::field_real) graph::deep_copy(execution_space_, modal_real_, 0.0); else graph::deep_copy(execution_space_, modal_complex_, Complex{}); } } } - if (problem_.raman) { + if (problem_.raman.enabled) { const auto started = std::chrono::steady_clock::now(); raman_drive_forward_plan_ = std::make_unique( raman_execution_space_, node_raman_drive_, node_raman_spectrum_, @@ -4925,7 +5011,7 @@ void GeneralWorkspace::plan() { raman_convolution_inverse_plan_ = std::make_unique( raman_execution_space_, node_raman_spectrum_, node_raman_convolved_, KokkosFFT::Direction::backward, axis); - const auto tail = problem_.n_nodes % problem_.node_chunk; + const auto tail = problem_.dims.n_nodes % problem_.dims.node_chunk; if (tail != 0) { // Reuse the leading rows of the full chunk buffers. A separate persistent // plan avoids transforming stale rows in the final partial node chunk. @@ -4960,12 +5046,12 @@ void GeneralWorkspace::plan() { std::chrono::duration(std::chrono::steady_clock::now() - started) .count(); } - if (!problem_.split_step_absorber) + if (!problem_.absorber.split_step) return; if (split_rows_ != 0) { // In place on the state's rows; a later execution may name another row block or the // other of the state's two views, of the same extents. - const auto& state = problem_.low_storage ? current_ : advanced_; + const auto& state = problem_.stepper.low_storage ? current_ : advanced_; const auto plan_rows = [&](const std::size_t count, const KokkosFFT::Direction direction) { const ComplexView2 rows = @@ -4986,7 +5072,7 @@ void GeneralWorkspace::plan() { } return; } - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { coarse_inverse_complex_plan_ = std::make_unique( fft_space_for_rows(coarse_time_complex_), coarse_spectrum_, coarse_time_complex_, KokkosFFT::Direction::backward, axis); @@ -5002,9 +5088,10 @@ void GeneralWorkspace::plan() { KokkosFFT::Direction::forward, axis); } if (temporal_tail_rows_ != 0) { - const auto& space = fft_space_for(temporal_tail_rows_ * problem_.n_t, problem_.n_t); + const auto& space = + fft_space_for(temporal_tail_rows_ * problem_.dims.n_t, problem_.dims.n_t); const auto rows = coarse_spectrum_.extent(0); - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { coarse_tail_inverse_complex_.create(space, coarse_spectrum_, coarse_time_complex_, rows, temporal_tail_rows_, KokkosFFT::Direction::backward); @@ -5021,7 +5108,7 @@ void GeneralWorkspace::plan() { } // The tail plans' measured planning overwrote their rows. graph::deep_copy(execution_space_, coarse_spectrum_, Complex{}); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) graph::deep_copy(execution_space_, coarse_time_complex_, Complex{}); else graph::deep_copy(execution_space_, coarse_time_real_, 0.0); @@ -5031,18 +5118,18 @@ void GeneralWorkspace::plan() { void GeneralWorkspace::set_initial(const std::span field, const std::span noise) { require_open(); - require_extent(field.size(), problem_.n_columns() * problem_.n_omega, + require_extent(field.size(), problem_.n_columns() * problem_.dims.n_omega, "initial field"); if (!noise.empty()) require_extent(noise.size(), field.size(), "noise"); upload_span(execution_space_, crossing_ring_, current_, field); crossed_.from_host += field.size_bytes(); if (noise.empty()) { - if (problem_.modified_noise) + if (problem_.stepper.modified_noise) graph::deep_copy(execution_space_, noise_, Complex{}); has_noise_ = false; } else { - if (!problem_.modified_noise) + if (!problem_.stepper.modified_noise) throw std::invalid_argument( "a native noise field is valid only for modified noise"); upload_span(execution_space_, crossing_ring_, noise_, noise); @@ -5074,23 +5161,24 @@ void GeneralWorkspace::require_trial_scalars(const TrialScalars& scalars) const throw std::invalid_argument(std::string(name) + " contains an invalid value"); }; if (!scalars.kerr_density.empty()) - require_extent(scalars.kerr_density.size(), problem_.n_stages, "kerr_density"); + require_extent(scalars.kerr_density.size(), problem_.stepper.n_stages, + "kerr_density"); checked_values(scalars.kerr_density, "kerr_density", false, false); - if (problem_.plasma && - (problem_.n_channels > 1 || !scalars.plasma_densities.empty() || + if (problem_.plasma.enabled && + (problem_.plasma.n_channels > 1 || !scalars.plasma_densities.empty() || !scalars.plasma_preionisation.empty())) { - const auto count = - checked_product(problem_.n_stages, problem_.n_channels, "stage channel values"); + const auto count = checked_product( + problem_.stepper.n_stages, problem_.plasma.n_channels, "stage channel values"); require_extent(scalars.plasma_densities.size(), count, "plasma_densities"); require_extent(scalars.plasma_preionisation.size(), count, "plasma_preionisation"); checked_values(scalars.plasma_densities, "plasma_densities", true, false); checked_values(scalars.plasma_preionisation, "plasma_preionisation", true, true); } - if (problem_.raman && - (problem_.n_raman_components > 1 || !scalars.raman_densities.empty())) { + if (problem_.raman.enabled && + (problem_.raman.n_components > 1 || !scalars.raman_densities.empty())) { require_extent(scalars.raman_densities.size(), - checked_product(problem_.n_stages, problem_.n_raman_components, - "stage Raman values"), + checked_product(problem_.stepper.n_stages, + problem_.raman.n_components, "stage Raman values"), "raman_densities"); checked_values(scalars.raman_densities, "raman_densities", true, false); } @@ -5103,29 +5191,32 @@ void GeneralWorkspace::require_trial_scalars(const TrialScalars& scalars) const throw std::invalid_argument("transform_weights weight a general modal guide's " "moving maps, and this problem has none"); require_extent(scalars.phi_delta.size(), - (problem_.n_stages + 1) * problem_.n_phi_terms, "phi_delta"); - require_extent(scalars.phi_at.size(), problem_.n_stages * problem_.n_phi_terms, - "phi_at"); + (problem_.stepper.n_stages + 1) * problem_.linear.n_phi_terms, + "phi_delta"); + require_extent(scalars.phi_at.size(), + problem_.stepper.n_stages * problem_.linear.n_phi_terms, "phi_at"); require_extent(scalars.prefactor.size(), - problem_.n_stages * problem_.n_prefactor_terms, "prefactor"); - require_extent(scalars.density.size(), problem_.n_stages, "density"); - require_extent(scalars.taper.size(), problem_.n_stages, "taper"); - if (problem_.plasma) - require_extent(scalars.preionisation.size(), problem_.n_stages, "preionisation"); - if (problem_.raman) { + problem_.stepper.n_stages * problem_.linear.n_prefactor_terms, + "prefactor"); + require_extent(scalars.density.size(), problem_.stepper.n_stages, "density"); + require_extent(scalars.taper.size(), problem_.stepper.n_stages, "taper"); + if (problem_.plasma.enabled) + require_extent(scalars.preionisation.size(), problem_.stepper.n_stages, + "preionisation"); + if (problem_.raman.enabled) { require_extent(scalars.raman_rates.size(), - problem_.n_stages * problem_.n_raman_terms, "raman_rates"); - if (!std::all_of( - scalars.raman_rates.begin(), scalars.raman_rates.end(), - [](const double value) { return std::isfinite(value) && value >= 0.0; })) + problem_.stepper.n_stages * problem_.raman.n_terms, "raman_rates"); + if (!std::ranges::all_of(scalars.raman_rates, [](const double value) { + return std::isfinite(value) && value >= 0.0; + })) throw std::invalid_argument("Raman decay rates must be finite and non-negative"); } } // Point the per-stage scalar views at a slot of the packed buffer. void GeneralWorkspace::bind_stage_scalars(const std::size_t slot) { - const auto stages = problem_.n_stages; - const auto channels = problem_.plasma ? problem_.n_channels : 0; + const auto stages = problem_.stepper.n_stages; + const auto channels = problem_.plasma.enabled ? problem_.plasma.n_channels : 0; double* const base = stage_scalars_.data() + slot * stage_slot_size_; const auto region = [&](const std::size_t index) { return base + stage_offsets_[index]; @@ -5144,11 +5235,11 @@ void GeneralWorkspace::bind_stage_scalars(const std::size_t slot) { stage_channel_preionisation_ = RealView2(region(8), stages, channels); stage_taper_ = RealView(region(9), stages); stage_preionisation_ = RealView(region(10), stages); - if (problem_.raman) { - stage_raman_rates_ = RealView2(region(11), stages, problem_.n_raman_terms); - if (problem_.n_raman_components > 1) + if (problem_.raman.enabled) { + stage_raman_rates_ = RealView2(region(11), stages, problem_.raman.n_terms); + if (problem_.raman.n_components > 1) stage_raman_densities_ = - RealView2(region(12), stages, problem_.n_raman_components); + RealView2(region(12), stages, problem_.raman.n_components); } // A compensated trial's rate weights are its step's, in slot 0 whichever slot the // stage views read. @@ -5160,7 +5251,7 @@ void GeneralWorkspace::bind_stage_scalars(const std::size_t slot) { // weights repeat the stage before's, and the dispersive response's densities. void GeneralWorkspace::note_trial_scalars(const TrialScalars& scalars) { // Rows of equal weights give bitwise equal phases (a zero weight's term is skipped). - const auto n_phi = problem_.n_phi_terms; + const auto n_phi = problem_.linear.n_phi_terms; const auto same_row = [&](const std::size_t row) { return std::equal( scalars.phi_delta.begin() + static_cast(row * n_phi), @@ -5168,10 +5259,10 @@ void GeneralWorkspace::note_trial_scalars(const TrialScalars& scalars) { scalars.phi_delta.begin() + static_cast((row - 1) * n_phi)); }; trial_taper_.assign(scalars.taper.begin(), scalars.taper.end()); - phase_repeats_.assign(problem_.n_stages, false); - for (std::size_t stage = 1; stage < problem_.n_stages; ++stage) + phase_repeats_.assign(problem_.stepper.n_stages, false); + for (std::size_t stage = 1; stage < problem_.stepper.n_stages; ++stage) phase_repeats_[stage] = same_row(stage); - whole_repeats_ = problem_.n_stages > 0 && same_row(problem_.n_stages); + whole_repeats_ = problem_.stepper.n_stages > 0 && same_row(problem_.stepper.n_stages); entered_stage_ = std::numeric_limits::max(); if (auto* weights = dispersive_weights()) weights->set_densities(scalars.dispersive_densities); @@ -5203,7 +5294,7 @@ void GeneralWorkspace::upload_trial(const TrialScalars& scalars, const std::vector& source) { if (source.size() != stage_offsets_[region + 1] - stage_offsets_[region]) throw std::invalid_argument("fixed problem array has the wrong extent"); - std::copy(source.begin(), source.end(), base + stage_offsets_[region]); + std::ranges::copy(source, base + stage_offsets_[region]); }; const auto fill = [&](const std::size_t region, const double value) { std::fill(base + stage_offsets_[region], base + stage_offsets_[region + 1], @@ -5214,7 +5305,7 @@ void GeneralWorkspace::upload_trial(const TrialScalars& scalars, place(2, values.prefactor); place(3, values.density); place(4, values.kerr_density.empty() ? values.density : values.kerr_density); - if (problem_.plasma) { + if (problem_.plasma.enabled) { if (values.plasma_densities.empty()) { place(5, values.density); place(7, values.density); @@ -5224,33 +5315,33 @@ void GeneralWorkspace::upload_trial(const TrialScalars& scalars, place(8, values.plasma_preionisation); // The sole channel already has stage-major contiguous scalar vectors. // Multiple channels consume their own matrices and need no extracted copy. - if (problem_.n_channels == 1) { + if (problem_.plasma.n_channels == 1) { place(5, values.plasma_densities); place(10, values.plasma_preionisation); } } } - if (problem_.raman) { + if (problem_.raman.enabled) { if (values.raman_densities.empty()) place(6, values.density); - else if (problem_.n_raman_components == 1) + else if (problem_.raman.n_components == 1) place(6, values.raman_densities); else fill(6, 1.0); place(11, values.raman_rates); - if (problem_.n_raman_components > 1) + if (problem_.raman.n_components > 1) place(12, values.raman_densities); } place(9, values.taper); - if (problem_.plasma && values.plasma_preionisation.empty()) + if (problem_.plasma.enabled && values.plasma_preionisation.empty()) place(10, values.preionisation); }; fill_slot(0, scalars); if (sample != nullptr) fill_slot(1, *sample); if (rate != nullptr) { - require_extent(rate->size(), problem_.n_phi_terms, "Adams rate weights"); - std::copy(rate->begin(), rate->end(), staging.data() + stage_offsets_[13]); + require_extent(rate->size(), problem_.linear.n_phi_terms, "Adams rate weights"); + std::ranges::copy(*rate, staging.data() + stage_offsets_[13]); } graph::deep_copy(execution_space_, stage_scalars_, staging); bind_stage_scalars(0); @@ -5266,22 +5357,23 @@ void GeneralWorkspace::use_uploaded_sample(const TrialScalars& sample) { void GeneralWorkspace::prepare_raman(const std::vector& rates, const std::vector& densities) { require_open(); - if (!problem_.raman) + if (!problem_.raman.enabled) throw std::logic_error("this state has no Raman workspace"); - require_extent(rates.size(), problem_.n_raman_terms, "Raman decay rates"); - if (!std::all_of(rates.begin(), rates.end(), [](const double value) { + require_extent(rates.size(), problem_.raman.n_terms, "Raman decay rates"); + if (!std::ranges::all_of(rates, [](const double value) { return std::isfinite(value) && value >= 0.0; })) throw std::invalid_argument("Raman decay rates must be finite and non-negative"); auto host = Kokkos::create_mirror_view(stage_raman_rates_); - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) host(0, term) = rates[term]; graph::deep_copy(execution_space_, stage_raman_rates_, host); - if (problem_.n_raman_components > 1) { - require_extent(densities.size(), problem_.n_raman_components, + if (problem_.raman.n_components > 1) { + require_extent(densities.size(), problem_.raman.n_components, "Raman amplitude densities"); - if (!std::all_of(densities.begin(), densities.end(), - [](double value) { return std::isfinite(value) && value >= 0.0; })) + if (!std::ranges::all_of(densities, [](double value) { + return std::isfinite(value) && value >= 0.0; + })) throw std::invalid_argument( "Raman amplitude densities must be finite and non-negative"); auto density_host = Kokkos::create_mirror_view(stage_raman_densities_); @@ -5296,19 +5388,19 @@ void GeneralWorkspace::prepare_raman(const std::vector& rates, void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double* rates, const double* densities) { bool unchanged = raman_kernel_ready_; - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) unchanged = unchanged && cached_raman_rates_[term] == rates[term]; - if (problem_.n_raman_components > 1) - for (std::size_t component = 0; component < problem_.n_raman_components; + if (problem_.raman.n_components > 1) + for (std::size_t component = 0; component < problem_.raman.n_components; ++component) unchanged = unchanged && cached_raman_densities_[component] == densities[component]; if (unchanged) return; - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) cached_raman_rates_[term] = rates[term]; - if (problem_.n_raman_components > 1) - for (std::size_t component = 0; component < problem_.n_raman_components; + if (problem_.raman.n_components > 1) + for (std::size_t component = 0; component < problem_.raman.n_components; ++component) cached_raman_densities_[component] = densities[component]; @@ -5320,12 +5412,12 @@ void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double const auto amplitudes = stage_raman_densities_; const auto row_component = raman_row_component_; const auto fractions = raman_component_fraction_; - const bool multiple = problem_.n_raman_components > 1; + const bool multiple = problem_.raman.n_components > 1; const bool tensor = problem_.effective_node_pol() == 2; - const auto terms = problem_.n_raman_terms; - const auto doubled = 2 * problem_.n_to; - const double fraction = problem_.raman_fraction; - const double dt = problem_.dt_fine; + const auto terms = problem_.raman.n_terms; + const auto doubled = 2 * problem_.dims.n_to; + const double fraction = problem_.raman.fraction; + const double dt = problem_.grid.dt_fine; launch::parallel_for( "solprop_native.raman.kernel_rebuild", Range2(raman_rebuild_space_, {0, 0}, {problem_.raman_kernels(), doubled}), @@ -5363,7 +5455,8 @@ void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double const auto spectrum = raman_kernel_spectrum_; launch::parallel_for( "solprop_native.raman.kernel_dt_scale", - Range2(raman_scale_space_, {0, 0}, {problem_.raman_kernels(), problem_.n_to + 1}), + Range2(raman_scale_space_, {0, 0}, + {problem_.raman_kernels(), problem_.dims.n_to + 1}), KOKKOS_LAMBDA(const std::size_t channel, const std::size_t index) { spectrum(channel, index) *= dt; }); @@ -5375,11 +5468,11 @@ void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double void GeneralWorkspace::prepare_trial(const double dz, const TrialScalars& scalars) { require_open(); - if (problem_.low_storage) - throw std::logic_error( - "trial_step: a low-storage state of " + std::to_string(current_.extent(0)) + - " x " + std::to_string(current_.extent(1)) + - " holds no Runge-Kutta views; its steps are low_storage_step's"); + if (problem_.stepper.low_storage) + refuse("trial_step: a low-storage state of ", current_.extent(0), + " x ", current_.extent(1), + " holds no Runge-Kutta views; its steps are " + "low_storage_step's"); if (!initialized_) throw std::logic_error("no initial field has been loaded"); require_trial_scalars(scalars); @@ -5399,16 +5492,16 @@ void GeneralWorkspace::prepare_trial(const double dz, const TrialScalars& scalar finish_pending_ = false; const bool deferrable = !flow_ && !problem_.separable_geometry(); defer_quadrature_ = deferred_quadrature_eligible(); - for (std::size_t stage = first; stage < problem_.n_stages; ++stage) { + for (std::size_t stage = first; stage < problem_.stepper.n_stages; ++stage) { try { - if (problem_.raman) + if (problem_.raman.enabled) prepare_raman_stage( - stage, scalars.raman_rates.data() + stage * problem_.n_raman_terms, - problem_.n_raman_components > 1 - ? scalars.raman_densities.data() + stage * problem_.n_raman_components + stage, scalars.raman_rates.data() + stage * problem_.raman.n_terms, + problem_.raman.n_components > 1 + ? scalars.raman_densities.data() + stage * problem_.raman.n_components : nullptr); - defer_finish_ = deferrable && stage + 1 < problem_.n_stages; - project_coarse_ = !defer_quadrature_ || stage + 1 == problem_.n_stages; + defer_finish_ = deferrable && stage + 1 < problem_.stepper.n_stages; + project_coarse_ = !defer_quadrature_ || stage + 1 == problem_.stepper.n_stages; evaluate_stage(stage, dz); } catch (...) { defer_finish_ = false; @@ -5442,23 +5535,24 @@ std::vector GeneralWorkspace::trial_step(const double dz, const auto in_norm = norm_mask_; const auto partials = partials_; const auto count = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto slots = reduction_slots_; accept_update( execution_space_, count, n, dz, KOKKOS_LAMBDA(const std::size_t column) { const auto slot = column * slots; - return UpdateColumn{current.data() + column * current.stride(0), - solution.data() + column * solution.stride(0), - error.data() + column * error.stride(0), - whole.data() + column * whole.stride(0), - advanced.data() + column * advanced.stride(0), - residual.data() + column * residual.stride(0), - in_norm.data(), - partials.data() + slot, - partials.data() + partials.stride(0) + slot, - partials.data() + 2 * partials.stride(0) + slot, - carried.data() + column * carried.stride(0)}; + return UpdateColumn{.current = current.data() + column * current.stride(0), + .solution = solution.data() + column * solution.stride(0), + .error = error.data() + column * error.stride(0), + .whole = whole.data() + column * whole.stride(0), + .advanced = advanced.data() + column * advanced.stride(0), + .residual = residual.data() + column * residual.stride(0), + .in_norm = in_norm.data(), + .old_partial = partials.data() + slot, + .new_partial = partials.data() + partials.stride(0) + slot, + .error_partial = + partials.data() + 2 * partials.stride(0) + slot, + .carried = carried.data() + column * carried.stride(0)}; }, "solprop_native.step.accept_error_partials", slots); return collect_trial(); @@ -5482,24 +5576,27 @@ std::vector GeneralWorkspace::adams_source(const std::vector& // does. Beside a coupled flow the sample is the material source alone (solprop SEP // 16 phase 5): the exponential formulation carries the residual in its compensation // and the flow in its moments, so the stage runs without the flow's pipeline. - require_extent(input.size(), problem_.n_columns() * problem_.n_omega, "Adams input"); + require_extent(input.size(), problem_.n_columns() * problem_.dims.n_omega, + "Adams input"); // The flow is set aside for the evaluation and put back whatever happens. auto held_flow = std::exchange(flow_, nullptr); + // An aggregate, so it declares no copy or move; it is never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions): a local scope guard struct RestoreFlow { std::unique_ptr& slot; std::unique_ptr& held; // NOLINTNEXTLINE(modernize-use-equals-default): restores the slot (see FlowAside). ~RestoreFlow() { slot = std::move(held); } - } restore{flow_, held_flow}; + } restore{.slot = flow_, .held = held_flow}; require_trial_scalars(scalars); upload_trial(scalars); if (general_) general_->set_weights(scalars.transform_weights); graph::deep_copy(execution_space_, advanced_, current_); copy_vector(execution_space_, current_, input); - if (problem_.raman) + if (problem_.raman.enabled) prepare_raman_stage(0, scalars.raman_rates.data(), - problem_.n_raman_components > 1 ? scalars.raman_densities.data() + problem_.raman.n_components > 1 ? scalars.raman_densities.data() : nullptr); evaluate_stage_source(0, 0.0); graph::deep_copy(execution_space_, current_, advanced_); @@ -5516,7 +5613,8 @@ void GeneralWorkspace::adopt(const std::vector& field, const double dz) require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - require_extent(field.size(), problem_.n_columns() * problem_.n_omega, "Adams state"); + require_extent(field.size(), problem_.n_columns() * problem_.dims.n_omega, + "Adams state"); copy_vector(execution_space_, advanced_, field); finish(dz); // The stash a response pass left is the predicted field's, not this one's. @@ -5691,15 +5789,15 @@ double GeneralWorkspace::low_storage_step(const double dz, const TrialScalars& s require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - if (!problem_.low_storage) + if (!problem_.stepper.low_storage) throw std::logic_error("this native state was not built for the low-storage march"); if (!(dz > 0.0) || !std::isfinite(dz)) throw std::invalid_argument("the low-storage step must be positive and finite"); - if (problem_.n_stages < 3) + if (problem_.stepper.n_stages < 3) throw std::invalid_argument("the low-storage march reads three scalar rows"); - if (rebuild_table && problem_.low_storage_flows != 1) + if (rebuild_table && problem_.stepper.low_storage_flows != 1) throw std::logic_error("only a constant operator's table is rebuilt"); - if (problem_.low_storage_flows == 1 && !rebuild_table && !table_ready_) + if (problem_.stepper.low_storage_flows == 1 && !rebuild_table && !table_ready_) throw std::logic_error("the low-storage table has not been built since the initial " "field was loaded"); require_trial_scalars(scalars); @@ -5711,11 +5809,11 @@ double GeneralWorkspace::low_storage_step(const double dz, const TrialScalars& s table_ready_ = true; for (std::size_t pass = 0; pass < 4; ++pass) { const auto row = low_storage_rows[pass]; - if (problem_.raman) + if (problem_.raman.enabled) prepare_raman_stage( - row, scalars.raman_rates.data() + row * problem_.n_raman_terms, - problem_.n_raman_components > 1 - ? scalars.raman_densities.data() + row * problem_.n_raman_components + row, scalars.raman_rates.data() + row * problem_.raman.n_terms, + problem_.raman.n_components > 1 + ? scalars.raman_densities.data() + row * problem_.raman.n_components : nullptr); if (problem_.separable_geometry()) low_storage_separable_pass(pass, dz); @@ -5750,10 +5848,10 @@ double GeneralWorkspace::low_storage_step(const double dz, const TrialScalars& s // formed in the finishes that apply them and hold nothing. void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const auto flows = low_storage_flows_; - const auto count = problem_.low_storage_flows; + const auto count = problem_.stepper.low_storage_flows; if (count == 0 || (count == 1 && !rebuild_table)) return; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // With vector maths a host task forms a column's exponents a block of bins at a // time and their exponentials in one pass (solprop ADR-059); a separable route's @@ -5776,7 +5874,8 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const std::size_t width = columns - first < stage_block ? columns - first : stage_block; const bool in_band = active(omega) != 0U; - double real[stage_block], imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; for (std::size_t offset = 0; offset < width; ++offset) { const Complex x = delta(2) * @@ -5792,7 +5891,7 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { return; } const auto terms = stage_terms(0); - const auto n_phi = problem_.n_phi_terms; + const auto n_phi = problem_.linear.n_phi_terms; const std::size_t blocks = (n + stage_block - 1) / stage_block; launch::parallel_for( "solprop_native.low_storage.modal_flows", @@ -5802,16 +5901,17 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const std::size_t column = item / blocks; const std::size_t first = (item % blocks) * stage_block; const std::size_t bins = n - first < stage_block ? n - first : stage_block; - double real[stage_block], imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; const auto form = [&](const std::size_t flow, const std::size_t offset) { terms.phase_block(offset, column, first, bins, real, imag); vmath::exp_complex(real, imag, real, imag, bins); for (std::size_t bin = 0; bin < bins; ++bin) flows(flow, column, first + bin) = Complex(real[bin], imag[bin]); }; - if (count == 1) + if (count == 1) { form(0, 2 * n_phi); - else { + } else { form(0, 0); form(1, n_phi); } @@ -5823,7 +5923,7 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const auto spatial = spatial_; const auto delta = stage_phi_delta_; const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; const auto index = register_index(); cartesian_for( "solprop_native.low_storage.separable_table", execution_space_, @@ -5835,7 +5935,7 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { return; } const auto terms = stage_terms(0); - const auto n_phi = problem_.n_phi_terms; + const auto n_phi = problem_.linear.n_phi_terms; cartesian_for( "solprop_native.low_storage.modal_flows", execution_space_, problem_.n_columns(), n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { @@ -5858,10 +5958,10 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { // output out of the scratch and re-forming the input (SEP 24 section 4.3). void GeneralWorkspace::low_storage_separable_pass(const std::size_t pass, const double dz) { - const bool material = problem_.nonlinear || problem_.plasma || problem_.raman || - separable_dispersive_ != nullptr; - const bool absorber = problem_.in_stage_absorber; - const bool elided = !problem_.low_storage_stage_register; + const bool material = problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || separable_dispersive_ != nullptr; + const bool absorber = problem_.absorber.in_stage; + const bool elided = !problem_.stepper.low_storage_stage_register; const auto row = low_storage_rows[pass]; if ((material || absorber) && (pass == 0 || !elided)) low_storage_separable_input(pass); @@ -5879,6 +5979,7 @@ void GeneralWorkspace::low_storage_separable_pass(const std::size_t pass, } } +namespace { // Each column's masked squared norm of `state` into `partials(0, column)`, in a team // per column: a device's final fold traverses like the others and reduces here, over // one volume, instead of reducing inside the fold's crossing (SEP 28). @@ -5904,6 +6005,7 @@ void column_norms(const char* label, const ExecutionSpace& execution, Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); }); } +} // namespace void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { const auto state = current_; @@ -5914,7 +6016,7 @@ void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { const auto active = active_mask_; const auto noise = noise_; const auto scratch = transverse_; - const bool add_noise = problem_.modified_noise && has_noise_; + const bool add_noise = problem_.stepper.modified_noise && has_noise_; const double scale = inverse_transverse_count(); const auto row = low_storage_rows[pass]; const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { @@ -5929,12 +6031,12 @@ void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { return value * scale; }; // In the volume route's order the scratch is the state's, so the input streams. - if (problem_.low_storage_tiled_input && !volume_order_) { + if (problem_.stepper.low_storage_tiled_input && !volume_order_) { // Read along the state's bins and written along the scratch's rows, a tile at a // time. tiled_crossing( "solprop_native.low_storage.separable_input", execution_space_, - problem_.n_columns(), problem_.n_omega, false, input, + problem_.n_columns(), problem_.dims.n_omega, false, input, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, const Complex value) { scratch.data()[index.cell(column, omega)] = value; @@ -5943,7 +6045,7 @@ void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { } crossing_for( "solprop_native.low_storage.separable_input", execution_space_, - problem_.n_columns(), problem_.n_omega, + problem_.n_columns(), problem_.dims.n_omega, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { scratch.data()[index.cell(column, omega)] = input(column, omega); }); @@ -5956,7 +6058,7 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, const auto accumulator = register_accumulator_; const auto stage_register = register_stage_; const auto flows = low_storage_flows_; - const bool tabulated = problem_.low_storage_flows != 0; + const bool tabulated = problem_.stepper.low_storage_flows != 0; const auto scratch = transverse_; const auto spatial = spatial_; const auto window = omega_window_; @@ -5966,16 +6068,16 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, const auto delta = stage_phi_delta_; const auto at = stage_phi_at_; // A device gathers and scatters the scratch through its own tiles. - [[maybe_unused]] const auto n_pol = problem_.n_pol_state; + [[maybe_unused]] const auto n_pol = problem_.dims.n_pol_state; [[maybe_unused]] const auto layout = transverse_layout(); const auto index = stage_index(); - const bool add_noise = problem_.modified_noise && has_noise_; + const bool add_noise = problem_.stepper.modified_noise && has_noise_; const double scale = inverse_transverse_count(); const int stage = static_cast(pass); const bool exchange = mode == fold_exchange; const bool second = mode == fold_second; // Elided, each of the first three passes writes the next pass's input in place of C. - const bool forward = !problem_.low_storage_stage_register && pass < 3; + const bool forward = !problem_.stepper.low_storage_stage_register && pass < 3; const bool final = pass == 3 && !exchange; // What this fold reads and writes, invariant over the traversal: each moves only the // registers its row of the flow table names. @@ -6052,7 +6154,7 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, if (volume_order_) cartesian_for( "solprop_native.low_storage.separable_fold", execution_space_, - problem_.n_columns(), problem_.n_omega, + problem_.n_columns(), problem_.dims.n_omega, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { static_cast(body(column, omega, scratch.data()[index.cell(column, omega)], @@ -6061,7 +6163,7 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, else tiled_update( "solprop_native.low_storage.separable_fold", execution_space_, - problem_.n_columns(), problem_.n_omega, exchange || forward, + problem_.n_columns(), problem_.dims.n_omega, exchange || forward, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { return layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol); }, @@ -6084,7 +6186,7 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, constexpr std::size_t block = crossing_tile_columns; constexpr std::size_t bins = crossing_tile_bins; const std::size_t columns = problem_.n_columns(); - const std::size_t n = problem_.n_omega; + const std::size_t n = problem_.dims.n_omega; const std::size_t blocks = (columns + block - 1) / block; const auto partials = partials_; launch::parallel_for( @@ -6135,9 +6237,10 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, void GeneralWorkspace::low_storage_modal_pass(const std::size_t pass, const double dz) { const auto row = low_storage_rows[pass]; const bool responds = general_ - ? general_->responds() || problem_.in_stage_absorber - : problem_.nonlinear || problem_.plasma || problem_.raman || - problem_.in_stage_absorber || dispersive_ != nullptr; + ? general_->responds() || problem_.absorber.in_stage + : problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || + problem_.absorber.in_stage || dispersive_ != nullptr; if (responds) low_storage_modal_input(pass); const bool evaluated = modal_core(row); @@ -6146,10 +6249,10 @@ void GeneralWorkspace::low_storage_modal_pass(const std::size_t pass, const doub const auto accumulator = register_accumulator_; const auto stage_register = register_stage_; const auto flows = low_storage_flows_; - const auto count = problem_.low_storage_flows; + const auto count = problem_.stepper.low_storage_flows; const auto in_norm = norm_mask_; - const auto n_phi = problem_.n_phi_terms; - const auto n_prefactor = problem_.n_prefactor_terms; + const auto n_phi = problem_.linear.n_phi_terms; + const auto n_prefactor = problem_.linear.n_prefactor_terms; const int stage = static_cast(pass); const bool final = pass == 3; const bool flows_here = stage != 1; @@ -6157,7 +6260,7 @@ void GeneralWorkspace::low_storage_modal_pass(const std::size_t pass, const doub const double step = dz; register_traversal( "solprop_native.low_storage.modal_fold", execution_space_, problem_.n_columns(), - problem_.n_omega, partials_, final, + problem_.dims.n_omega, partials_, final, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { Complex source{}; if (evaluated) { @@ -6198,7 +6301,7 @@ void GeneralWorkspace::low_storage_modal_input(const std::size_t pass) { const auto source = pass == 0 ? current_ : register_stage_; const auto row = low_storage_rows[pass]; const auto terms = stage_terms(row); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto gaps = fine_spectrum_size() - n; cartesian_for( "solprop_native.low_storage.modal_input", execution_space_, problem_.n_columns(), @@ -6241,7 +6344,7 @@ void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compen if (compensated && problem_.separable_geometry()) throw std::logic_error("a separable operator needs no compensation"); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; if (compensated && adams_states_.extent(0) != slots) { adams_states_ = ComplexView3("solprop_native.adams.states", slots, columns, n); adams_rates_ = ComplexView3("solprop_native.adams.rates", slots, columns, n); @@ -6251,7 +6354,7 @@ void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compen adams_endpoint_rate_ = ComplexView2("solprop_native.adams.endpoint_rate", columns, n); adams_rate_weights_ = Kokkos::View( - "solprop_native.adams.rate_weights", problem_.n_phi_terms); + "solprop_native.adams.rate_weights", problem_.linear.n_phi_terms); } // Dense observations are opt-in. Keeping their slopes and saved fields on a run // that lands on its planes costs (slots + 4) whole fields without reading them. @@ -6293,17 +6396,17 @@ void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compen adams_fallback_ = {}; else if (adams_fallback_.extent(0) != rows || adams_fallback_.extent(1) != blocks) adams_fallback_ = Kokkos::View( - "solprop_native.adams.fallback_functions", rows, blocks, problem_.n_modes, - problem_.n_modes); - if (columns != problem_.n_modes) + "solprop_native.adams.fallback_functions", rows, blocks, + problem_.dims.n_modes, problem_.dims.n_modes); + if (columns != problem_.dims.n_modes) throw std::logic_error("the resident Adams engine steps a coupled block of one " "batch row"); #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // A wider block's host passes hold its channel vectors in a row per bin; a // device's passes take a thread per (bin, channel) and hold none, at any width // (solprop SEP 47 phase 4). - const std::size_t width = problem_.n_modes > adams_max_channels - ? (adams_max_samples + 1) * problem_.n_modes + const std::size_t width = problem_.dims.n_modes > adams_max_channels + ? (adams_max_samples + 1) * problem_.dims.n_modes : 0; if (adams_channels_.extent(1) != width) adams_channels_ = width == 0 @@ -6312,8 +6415,8 @@ void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compen #endif if (adams_block_.extent(0) != slots + 1) adams_block_ = Kokkos::View( - "solprop_native.adams.block", slots + 1, flow_->n_active(), problem_.n_modes, - problem_.n_modes); + "solprop_native.adams.block", slots + 1, flow_->n_active(), + problem_.dims.n_modes, problem_.dims.n_modes); if (compensated && flow_->n_terms() != 0 && adams_applied_.extent(0) != slots) { adams_applied_ = ComplexView3("solprop_native.adams.applied", slots, columns, n); adams_end_applied_ = ComplexView2("solprop_native.adams.end_applied", columns, n); @@ -6374,7 +6477,7 @@ SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_phase_sum( real = real + weight * fixed.real(); imag = imag + weight * fixed.imag(); } - return Complex(real, imag); + return {real, imag}; } } // namespace @@ -6389,11 +6492,11 @@ void GeneralWorkspace::adams_phase() { const auto delta = stage_phi_delta_; const auto exponent = adams_exponent_; const auto propagator = adams_propagator_; - const auto n_pol = problem_.n_pol_state; - const auto n_stages = problem_.n_stages; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_stages = problem_.stepper.n_stages; launch::parallel_for( "solprop_native.adams.separable_phase", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { const auto values = spatial.value(column / n_pol, index, active(index) != 0U); const Complex x = delta(n_stages) * values.linear; @@ -6406,11 +6509,11 @@ void GeneralWorkspace::adams_phase() { const auto phi_delta = stage_phi_delta_; const auto exponent = adams_exponent_; const auto propagator = adams_propagator_; - const auto n_phi = problem_.n_phi_terms; - const auto offset = problem_.n_stages * n_phi; + const auto n_phi = problem_.linear.n_phi_terms; + const auto offset = problem_.stepper.n_stages * n_phi; launch::parallel_for( "solprop_native.adams.phase", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), // The step's phase increment: strict (solprop SEP 25 section 4.2). KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) SOLPROP_NATIVE_STRICT_FUNCTION { @@ -6427,9 +6530,9 @@ void GeneralWorkspace::adams_phase() { void GeneralWorkspace::adams_evaluate_source(const TrialScalars& scalars) { if (general_) general_->set_weights(scalars.transform_weights); - if (problem_.raman) + if (problem_.raman.enabled) prepare_raman_stage(0, scalars.raman_rates.data(), - problem_.n_raman_components > 1 ? scalars.raman_densities.data() + problem_.raman.n_components > 1 ? scalars.raman_densities.data() : nullptr); evaluate_stage_source(0, 0.0, false); stages_ready_ = true; @@ -6459,7 +6562,7 @@ void GeneralWorkspace::adams_start(const std::size_t slot, // side of a kink that restarted the history (solprop SEP 16 phase 6). if (adams_applied_.extent(0) != 0) { upload_weights(execution_space_, adams_end_weights_, scalars.flow_weights); - apply_generator(execution_space_, problem_.n_omega, problem_.n_modes, + apply_generator(execution_space_, problem_.dims.n_omega, problem_.dims.n_modes, flow_->block_slots(), flow_->residual_terms(), adams_end_weights_, current_, Kokkos::subview(adams_applied_, slot, Kokkos::ALL, Kokkos::ALL)); @@ -6468,7 +6571,7 @@ void GeneralWorkspace::adams_start(const std::size_t slot, const auto store = adams_store_; launch::parallel_for( "solprop_native.adams.start", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { store(slot, column, index) = stages(0, column, index); }); @@ -6478,7 +6581,7 @@ void GeneralWorkspace::adams_scale(const double rtol, const double atol, const std::vector& weights) { require_open(); if (!weights.empty()) - require_extent(weights.size(), problem_.n_omega, "Adams scale weights"); + require_extent(weights.size(), problem_.dims.n_omega, "Adams scale weights"); adams_rtol_ = rtol; adams_atol_ = atol; adams_scale_weights_ = RealView("solprop_native.adams.scale_weights", weights.size()); @@ -6492,17 +6595,17 @@ void GeneralWorkspace::adams_scale(const double rtol, const double atol, // A trial's divisor: the host's when one is given, else formed on the device from the // accepted state, as solprop's PIController.scale forms it from the same magnitude. void GeneralWorkspace::adams_upload_divisor(const std::vector& divisor) { - if (adams_divisor_.extent(0) != problem_.n_omega) + if (adams_divisor_.extent(0) != problem_.dims.n_omega) adams_divisor_ = Kokkos::View( - "solprop_native.adams.divisor", problem_.n_omega); + "solprop_native.adams.divisor", problem_.dims.n_omega); if (!divisor.empty() || !adams_device_scale_) { - require_extent(divisor.size(), problem_.n_omega, "Adams norm divisor"); + require_extent(divisor.size(), problem_.dims.n_omega, "Adams norm divisor"); copy_vector(execution_space_, adams_divisor_, divisor); return; } const auto current = current_; const auto in_norm = norm_mask_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; // A threaded reduction in whatever order measures fastest (solprop SEP 25). launch::parallel_reduce( "solprop_native.adams.magnitude", @@ -6534,7 +6637,7 @@ double GeneralWorkspace::adams_magnitude() { require_adams(); const auto current = current_; const auto in_norm = norm_mask_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; double total = 0.0; launch::parallel_reduce( "solprop_native.adams.magnitude", @@ -6617,7 +6720,7 @@ GeneralWorkspace::adams_fused_trial(const double dz, const AdamsWeights& weights adams_upload_divisor(divisor); adams_phase(); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); const auto store = adams_store_; const auto current = current_; @@ -6650,9 +6753,15 @@ GeneralWorkspace::adams_fused_trial(const double dz, const AdamsWeights& weights advanced(column, index) = propagator(column, index) * (current(column, index) + dz * total); }); - const AdamsDefectNorms reducer{weights.n_rows, store, source, - propagator, norm_mask_, adams_divisor_, - weights, dz, n}; + const AdamsDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .source = source, + .propagator = propagator, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .n = n}; return adams_norms( "solprop_native.adams.defect_norms", Kokkos::RangePolicy(execution_space_, 0, columns * n), reducer, @@ -6671,7 +6780,7 @@ void GeneralWorkspace::adams_fused_accept(const double dz, const AdamsWeights& k throw std::logic_error("Adams dense observation storage is not allocated"); adams_trial_ready_ = false; const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); const auto store = adams_store_; const auto source = adams_source_; @@ -6825,8 +6934,9 @@ void GeneralWorkspace::table_upload(const TableKind kind, std::vector ed require_open(); const bool prefactor = kind == TableKind::prefactor; const bool flux = kind == TableKind::flux; - if (flux ? !problem_.flux_table - : (prefactor ? !problem_.prefactor_table : !problem_.phi_table)) + if (flux + ? !problem_.flux.table + : (prefactor ? !problem_.linear.prefactor_table : !problem_.linear.phi_table)) throw std::logic_error(std::string("this problem's ") + (flux ? "plane measure" : prefactor ? "prefactor" @@ -6835,13 +6945,12 @@ void GeneralWorkspace::table_upload(const TableKind kind, std::vector ed if (n_intervals == 0 || n_terms == 0 || edges.size() != n_intervals + 1) throw std::invalid_argument("a z table needs one more breakpoint than intervals " "and at least one coefficient"); - if (flux && n_values != problem_.n_omega) - throw std::invalid_argument("the plane measure's material table holds n_omega = " + - std::to_string(problem_.n_omega) + " values"); - if (!flux && n_values != problem_.n_columns() * problem_.n_omega) - throw std::invalid_argument("a z table's values must be the state's (" + - std::to_string(problem_.n_columns()) + " x " + - std::to_string(problem_.n_omega) + ")"); + if (flux && n_values != problem_.dims.n_omega) + refuse("the plane measure's material table holds n_omega = ", problem_.dims.n_omega, + " values"); + if (!flux && n_values != problem_.n_columns() * problem_.dims.n_omega) + refuse("a z table's values must be the state's (", problem_.n_columns(), " x ", + problem_.dims.n_omega, ")"); for (std::size_t index = 0; index < edges.size(); ++index) if (!std::isfinite(edges[index]) || (index > 0 && !(edges[index] > edges[index - 1]))) @@ -6880,10 +6989,10 @@ void GeneralWorkspace::table_evaluate(const std::vector& slots, if (!phi_table_.present() && !prefactor_table_.present()) throw std::logic_error("no z table has been uploaded"); for (const auto slot : slots) { - if ((phi_table_.present() && slot >= problem_.n_phi_terms) || - (prefactor_table_.present() && slot >= problem_.n_prefactor_terms)) - throw std::out_of_range("z table slot " + std::to_string(slot) + - " is not a term of this problem"); + if ((phi_table_.present() && slot >= problem_.linear.n_phi_terms) || + (prefactor_table_.present() && slot >= problem_.linear.n_prefactor_terms)) + refuse("z table slot ", slot, + " is not a term of this problem"); } if (phi_table_.present()) z_table::evaluate("solprop_native.linear.phi_table_values", execution_space_, @@ -6913,9 +7022,8 @@ void GeneralWorkspace::table_rate(const std::size_t slot, const double z, require_open(); if (!rate_table_.present()) throw std::logic_error("no rate table has been uploaded"); - if (slot >= problem_.n_phi_terms) - throw std::out_of_range("z table slot " + std::to_string(slot) + - " is not a term of this problem"); + if (slot >= problem_.linear.n_phi_terms) + refuse("z table slot ", slot, " is not a term of this problem"); z_table::evaluate("solprop_native.linear.rate_table_values", execution_space_, rate_table_, z_table::locate(rate_table_, {slot}, {z}, before), phi_terms_.data()); @@ -6927,8 +7035,7 @@ GeneralWorkspace::table_terms_to_host(const TableKind kind, require_open(); const auto& terms = kind == TableKind::prefactor ? prefactor_terms_ : phi_terms_; if (slot >= terms.extent(0)) - throw std::out_of_range("term slot " + std::to_string(slot) + - " is not a term of this problem"); + refuse("term slot ", slot, " is not a term of this problem"); execution_space_.fence("solprop_native z table terms_to_host"); const auto host = Kokkos::create_mirror_view_and_copy( Kokkos::HostSpace{}, Kokkos::subview(terms, slot, Kokkos::ALL, Kokkos::ALL)); @@ -6946,16 +7053,16 @@ GeneralWorkspace::phase_sum_to_host(const std::vector& weights) { require_open(); if (problem_.separable_geometry()) throw std::logic_error("a separable operator has no Phi terms"); - require_extent(weights.size(), problem_.n_phi_terms, "phase weights"); + require_extent(weights.size(), problem_.linear.n_phi_terms, "phase weights"); RealView row("solprop_native.diagnostic.phase_weights", weights.size()); copy_vector(execution_space_, row, weights); ComplexView2 phase("solprop_native.diagnostic.phase", problem_.n_columns(), - problem_.n_omega); + problem_.dims.n_omega); auto terms = stage_terms(0); terms.phi_delta = row; launch::parallel_for( "solprop_native.diagnostic.phase_sum", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { phase(column, index) = terms.phase_sum(0, column, index); }); @@ -6978,7 +7085,7 @@ void GeneralWorkspace::adams_observe(const AdamsWeights& weights, upload_trial(scalars); adams_phase(); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto count = adams_dense_count_; const auto start = adams_dense_start_; const auto exponent = adams_dense_exponent_; @@ -7115,35 +7222,12 @@ adams_device_predict(const ExponentialWeights& weights, const std::size_t formed const double dz, const Moments& moments, const Store& store, const std::size_t column, const std::size_t index) { static_assert(adams_max_samples == 8, "one case per sample count"); - switch (formed) { - case 1: - return adams_device_prediction<1>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 2: - return adams_device_prediction<2>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 3: - return adams_device_prediction<3>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 4: - return adams_device_prediction<4>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 5: - return adams_device_prediction<5>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 6: - return adams_device_prediction<6>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 7: - return adams_device_prediction<7>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 8: - return adams_device_prediction<8>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - default: - return adams_device_prediction<0>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - } + const auto predict = [&] { + return adams_device_prediction(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + }; + return dispatch_width<1, adams_max_samples>( + formed, predict, [&] { return predict.template operator()<0>(); }); } #endif @@ -7321,17 +7405,17 @@ struct CoupledDefectNorms { // translate.linop.rate_weights), into `out`. void GeneralWorkspace::adams_rate(const std::vector& rate, const ComplexView2& out) { - require_extent(rate.size(), problem_.n_phi_terms, "Adams rate weights"); + require_extent(rate.size(), problem_.linear.n_phi_terms, "Adams rate weights"); if (adams_rate_weights_.extent(0) != rate.size()) adams_rate_weights_ = Kokkos::View( "solprop_native.adams.rate_weights", rate.size()); copy_vector(execution_space_, adams_rate_weights_, rate); const auto weights = adams_rate_weights_; const auto phi_terms = phi_terms_; - const auto n_phi = problem_.n_phi_terms; + const auto n_phi = problem_.linear.n_phi_terms; launch::parallel_for( "solprop_native.adams.rate", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { Complex value{}; for (std::size_t term = 0; term < n_phi; ++term) @@ -7369,6 +7453,9 @@ void GeneralWorkspace::adams_compensation_start(const std::size_t slot, // `strict_scale(1, x)` is x, so its exponential is the same bits. A host block with // vector maths forms the same values on its lanes // (`adams_lanes_predicted_compensation`). +// The strict compensation helpers keep their linkage: changing it could change how GCC +// inlines strict code (fp.hpp), which needs its own A/B. +// NOLINTBEGIN(misc-use-internal-linkage): strict helpers, linkage unchanged template SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_predicted_compensation(const ExponentialWeights& weights, const Complex x, @@ -7441,17 +7528,14 @@ adams_observed_compensation(const ExponentialWeights& weights, const std::size_t return total; } +// NOLINTEND(misc-use-internal-linkage) namespace { // The elements a host task of the elementwise trial's passes takes in turn. Each pass // runs its loops over the block, so that what one loop writes the next reads from // cache, and a strict loop is one call a block on a compiler that keeps strict // functions out of line; a device takes a thread an element. -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -constexpr std::size_t adams_block = 1; -#else -constexpr std::size_t adams_block = adams_lane_width; -#endif +constexpr std::size_t adams_block = build::device ? 1 : adams_lane_width; // The host blocks run as lanes where the vector maths forms their exponentials in // passes (`vmath::vectorised`); elsewhere (devices, the system libm on Arm) each @@ -7479,7 +7563,7 @@ adams_phase_block(const Terms& phi_terms, const Weights& phi_delta, } } -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The history rows a pass reads, rotated by a carrier frame's factors // (`adams_history`). Strict. @@ -7565,7 +7649,8 @@ adams_lanes_predicted_compensation(AdamsLanes& lanes, const ExponentialWeights& lanes.ci[t][b] = ri + ir; } } - double total_r[AdamsLanes::B], total_i[AdamsLanes::B]; + double total_r[AdamsLanes::B]; + double total_i[AdamsLanes::B]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = 0.0; total_i[b] = 0.0; @@ -7628,7 +7713,8 @@ inline void adams_lanes_corrected_compensation(AdamsLanes& lanes, SOLPROP_NATIVE_STRICT_BLOCK constexpr std::size_t B = AdamsLanes::B; const std::size_t count = lanes.count; - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; const double weight0 = weights.comp_corrector[0]; for (std::size_t b = 0; b < count; ++b) { const double scaled_r = inverse_dz * lanes.xr[b]; @@ -7672,7 +7758,10 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei constexpr std::size_t B = AdamsLanes::B; const std::size_t count = lanes.count; for (std::size_t row = 0; row < scored; ++row) { - double difference_r[B], difference_i[B], weight_r[B], weight_i[B]; + double difference_r[B]; + double difference_i[B]; + double weight_r[B]; + double weight_i[B]; const double weight0 = weights.rows[row][0]; for (std::size_t b = 0; b < count; ++b) { difference_r[b] = weight0 * lanes.wr[b]; @@ -7698,7 +7787,8 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei weight_i[b] = weight_i[b] + imag; } } - double defect_r[B], defect_i[B]; + double defect_r[B]; + double defect_i[B]; for (std::size_t b = 0; b < count; ++b) { const double rr = difference_r[b] * weight_r[b]; const double ii = difference_i[b] * weight_i[b]; @@ -7710,7 +7800,8 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei defect_i[b] = dz * product_i; } if (weights.compensated) { - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; const double first = weights.comp_rows[row][0]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = first * lanes.nr[b]; @@ -7743,7 +7834,7 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei } #endif -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The whole step's phase increment `sum_t dPhi_t Phi_t` of every lane into `xr`/`xi`, // as `adams_phase_sum` forms it, and with `Rate` a varying operator's endpoint rate // `sum_t w_t Phi_t` into `dr`/`di`, as `adams_corrected_compensation` forms it, in the @@ -7792,7 +7883,7 @@ adams_lanes_phase(AdamsLanes& lanes, const Terms& phi_terms, const Weights& phi_ } #endif -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES /** * The predict pass on a host block's lanes, the elements [first, last) of a column * whose exponents are formed: the flow, the moments, the predictor's carried samples @@ -7877,7 +7968,8 @@ inline double adams_lanes_predict( adams_lanes_history(lanes, weights, store, weights.order, column, first); // The prediction exp(x) u + dz sum_k M_k sum_j c_jk F_j. const Complex* accepted = ¤t(column, first); - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < n; ++b) { lanes.ur[b] = accepted[b].real(); lanes.ui[b] = accepted[b].imag(); @@ -7885,7 +7977,8 @@ inline double adams_lanes_predict( total_i[b] = 0.0; } for (std::size_t k = 0; k < weights.order; ++k) { - double combined_r[B], combined_i[B]; + double combined_r[B]; + double combined_i[B]; for (std::size_t b = 0; b < n; ++b) { combined_r[b] = 0.0; combined_i[b] = 0.0; @@ -8006,7 +8099,7 @@ struct ExponentialCorrection { // The host lanes' predict pass formed the endpoint rate (off a separable route). bool rate_formed; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The pass on a host block's lanes (`AdamsLanes`), as the element-by-element form // below writes it; the norms' sums are taken in element order. void lanes_block(const std::size_t column, const std::size_t first, @@ -8031,13 +8124,15 @@ struct ExponentialCorrection { lanes.mi[k][b] = moment[b].imag(); } } - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = 0.0; total_i[b] = 0.0; } for (std::size_t k = 0; k < weights.count; ++k) { - double combined_r[B], combined_i[B]; + double combined_r[B]; + double combined_i[B]; const double weight0 = weights.corrector[0][k]; for (std::size_t b = 0; b < count; ++b) { combined_r[b] = weight0 * lanes.wr[b]; @@ -8200,7 +8295,7 @@ struct ExponentialCorrection { const auto first = (task % blocks) * adams_block; const auto last = first + adams_block < n ? first + adams_block : n; const double magnitude = device_scale ? Kokkos::sqrt(magnitude_sum()) : 0.0; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES if constexpr (vmath::vectorised) { lanes_block(column, first, last, magnitude, update); return; @@ -8209,26 +8304,13 @@ struct ExponentialCorrection { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) static_assert(adams_block == 1, "a device thread takes one element"); if (first < last) { - switch (weights.width) { - case 1: - return device_element<1>(column, first, magnitude, update); - case 2: - return device_element<2>(column, first, magnitude, update); - case 3: - return device_element<3>(column, first, magnitude, update); - case 4: - return device_element<4>(column, first, magnitude, update); - case 5: - return device_element<5>(column, first, magnitude, update); - case 6: - return device_element<6>(column, first, magnitude, update); - case 7: - return device_element<7>(column, first, magnitude, update); - case 8: - return device_element<8>(column, first, magnitude, update); - default: - break; - } + const auto element = [&] { + device_element(column, first, magnitude, update); + return true; + }; + if (dispatch_width<1, adams_max_samples>(weights.width, element, + [] { return false; })) + return; } #endif double scale[adams_block]{}; @@ -8291,7 +8373,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( const std::vector& divisor, const TrialScalars& step, const TrialScalars& sample, const std::vector& rate) { const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const bool compensated = weights.compensated; // solprop's own controller's scale is formed from the magnitude below; any other // controller's divisor crosses. @@ -8302,7 +8384,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( // weights cross, with the step's and the sample's scalars, in the trial's one copy. upload_trial(step, &sample, compensated ? &rate : nullptr); const std::size_t blocks = (n + adams_block - 1) / adams_block; -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA // A device thread takes one element of a long FP64 chain, and Kokkos's CUDA range // reduction gives each block 256 of them: a four-mode capillary's 10496 elements // were 41 blocks on an NVIDIA A40's 84 multiprocessors. Blocks of 128 take them @@ -8315,11 +8397,11 @@ std::vector GeneralWorkspace::adams_elementwise_trial( const bool separable = problem_.separable_geometry(); const auto spatial = spatial_; const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto n_stages = problem_.n_stages; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_stages = problem_.stepper.n_stages; const auto phi_terms = phi_terms_; const auto phi_delta = stage_phi_delta_; - const auto n_phi = problem_.n_phi_terms; + const auto n_phi = problem_.linear.n_phi_terms; const auto offset = n_stages * n_phi; const auto exponent = adams_exponent_; const auto flow = adams_propagator_; @@ -8335,7 +8417,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( const std::size_t count = weights.count; const std::size_t formed = count > weights.order ? count : weights.order; const double inverse_dz = 1.0 / dz; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // A host block's predict pass forms the corrector's endpoint rate beside the phase, // from the weights the trial uploaded with its scalars. const auto endpoint_rates = stage_rate_weights_; @@ -8372,7 +8454,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( #endif flow(column, index) = exponential(x); } -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES if constexpr (passes) { magnitude += separable ? adams_lanes_predict( @@ -8444,35 +8526,35 @@ std::vector GeneralWorkspace::adams_elementwise_trial( // On a device the source's launches replay as one graph while they repeat. adams_source_graph_.run(execution_space_, [&] { adams_evaluate_source(sample); }); std::swap(current_, adams_saved_); - const ExponentialCorrection reducer{weights.n_rows, - store, - moments, - stages_, - carried, - phi_terms, - adams_source_, - flow, - current_, - advanced_, - exponent, - lab, - adams_endpoint_, - adams_endpoint_rate_, - stage_rate_weights_, - in_norm, - adams_divisor_, - adams_scale_weights_, - adams_magnitude_sum_, - weights, - dz, - inverse_dz, - adams_rtol_, - adams_atol_, - n, - blocks, - n_phi, - device_scale, - !separable}; + const ExponentialCorrection reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .stages = stages_, + .carried = carried, + .phi_terms = phi_terms, + .source = adams_source_, + .flow = flow, + .current = current_, + .advanced = advanced_, + .exponent = exponent, + .lab = lab, + .endpoint = adams_endpoint_, + .endpoint_rate = adams_endpoint_rate_, + .rate_weights = stage_rate_weights_, + .in_norm = in_norm, + .divisor = adams_divisor_, + .scale_weights = adams_scale_weights_, + .magnitude_sum = adams_magnitude_sum_, + .weights = weights, + .dz = dz, + .inverse_dz = inverse_dz, + .rtol = adams_rtol_, + .atol = adams_atol_, + .n = n, + .blocks = blocks, + .n_phi = n_phi, + .device_scale = device_scale, + .rate_formed = !separable}; auto norms = adams_norms("solprop_native.adams.exponential_correct", tasks, reducer, weights.n_rows); adams_trial_compensated_ = compensated; @@ -8492,10 +8574,10 @@ void GeneralWorkspace::adams_fallback_squarings(const std::size_t section, const auto route = flow_->route(); const auto triangular = flow_->fallback_triangular(); const auto exponent = adams_exponent_; - const auto modes = problem_.n_modes; + const auto modes = problem_.dims.n_modes; const std::size_t count = weights.count; - const auto policy = - Kokkos::RangePolicy(flow_execution_space_, 0, problem_.n_omega); + const auto policy = Kokkos::RangePolicy(flow_execution_space_, 0, + problem_.dims.n_omega); // The moments' augmented exponent, and the carries' `span dz T`, apart, so that a // refusal names which. int moments = 0; @@ -8531,14 +8613,14 @@ void GeneralWorkspace::adams_fallback_squarings(const std::size_t section, return; std::array length{}; std::snprintf(length.data(), length.size(), "%.4g", dz); - const std::string limit = " would need more than " + - std::to_string(flow_functions::max_squarings) + - " squarings, the most this backend admits. Cap dz_max"; + const std::string limit = + concat(" would need more than ", flow_functions::max_squarings, + " squarings, the most this backend admits. Cap dz_max"); if (moments > flow_functions::max_squarings) - throw BackendRefusal(bin + " over a step of " + length.data() + " m: its exponent" + - limit); - throw BackendRefusal(bin + " over a step of " + length.data() + - " m: a history sample's carry to the step's end" + limit); + refuse(bin, " over a step of ", length.data(), " m: its exponent", + limit); + refuse(bin, " over a step of ", length.data(), + " m: a history sample's carry to the step's end", limit); } std::vector GeneralWorkspace::adams_exponential_trial( @@ -8574,7 +8656,7 @@ std::vector GeneralWorkspace::adams_exponential_trial( if (residual) upload_weights(execution_space_, adams_end_weights_, step.flow_weights); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); const auto bins = Kokkos::RangePolicy(flow_execution_space_, 0, n); const auto exponent = adams_exponent_; @@ -8601,7 +8683,7 @@ std::vector GeneralWorkspace::adams_exponential_trial( const auto block = adams_block_; // A host's channel vectors of a block wider than `adams_max_channels`. [[maybe_unused]] const auto channels = adams_channels_; - const auto modes = problem_.n_modes; + const auto modes = problem_.dims.n_modes; // The bins off the eigenbasis route (solprop SEP 47 phase 4). const auto route = flow_->route(); const auto fallback_vectors = flow_->fallback_vectors(); @@ -9261,10 +9343,20 @@ std::vector GeneralWorkspace::adams_exponential_trial( } }); #endif - const CoupledDefectNorms reducer{weights.n_rows, store, moments, source, - carried, endpoint, block, slots, - norm_mask_, adams_divisor_, weights, dz, - modes, adams_channels_}; + const CoupledDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .source = source, + .carried = carried, + .endpoint = endpoint, + .block = block, + .slots = slots, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .modes = modes, + .channels = adams_channels_}; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // A device item is a (bin, channel) pair, a host item a bin. const auto items = n * modes; @@ -9291,7 +9383,7 @@ void GeneralWorkspace::adams_exponential_accept(const double dz, const std::size throw std::logic_error("Adams dense observation storage is not allocated"); adams_trial_ready_ = false; const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); const auto store = adams_store_; const auto source = adams_source_; @@ -9339,7 +9431,7 @@ void GeneralWorkspace::adams_exponential_accept(const double dz, const std::size // half steps, or the band's mask in the pass below. Untransported samples: nothing // moves; the new sample takes row `free`, and for a varying operator its state and // rate the rows beside it. - const bool masked = !problem_.split_step_absorber; + const bool masked = !problem_.absorber.split_step; if (!masked) split_step(dz, advanced_); const bool compensated = adams_trial_compensated_; @@ -9385,9 +9477,8 @@ void GeneralWorkspace::adams_exponential_accept(const double dz, const std::size // tables while the device finishes the step (on an AMD Instinct MI325X a trial's // kernels take a few microseconds each, so the wait was a visible part of a trial). void GeneralWorkspace::adams_accepted_boundary() const { -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - execution_space_.fence("solprop_native Adams accepted-step boundary"); -#endif + if constexpr (build::host) + execution_space_.fence("solprop_native Adams accepted-step boundary"); } // The field at the fraction `xi` of the last accepted step: the accepted corrector's @@ -9399,7 +9490,7 @@ void GeneralWorkspace::adams_exponential_observe(const double xi, const double d if (adams_dense_count_ == 0) throw std::logic_error("no Adams step is held for dense observation"); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const std::size_t count = adams_dense_count_; const auto start = adams_dense_start_; const auto exponent = adams_dense_exponent_; @@ -9508,12 +9599,13 @@ struct StepExponent { void GeneralWorkspace::rk_dense_allocate(const std::vector& weights) { require_open(); - if (problem_.low_storage || stages_.extent(0) == 0) + if (problem_.stepper.low_storage || stages_.extent(0) == 0) throw std::logic_error("Runge-Kutta dense output needs the Runge-Kutta stages"); - if (weights.size() != problem_.n_stages || weights.size() > rk_dense_max_stages) + if (weights.size() != problem_.stepper.n_stages || + weights.size() > rk_dense_max_stages) throw std::invalid_argument("the dense-output weights are one per stage"); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; rk_dense_weights_ = weights; if (rk_dense_coefficients_.extent(0) == 0) { rk_dense_coefficients_ = @@ -9532,27 +9624,27 @@ void GeneralWorkspace::rk_dense_hold(const double dz) { if (!stages_ready_) throw std::logic_error("no Runge-Kutta trial is held for a dense extension"); RungeKuttaDenseWeights weights; - weights.stages = problem_.n_stages; + weights.stages = problem_.stepper.n_stages; for (std::size_t stage = 0; stage < weights.stages; ++stage) { - weights.solution[stage] = problem_.solution_weights[stage]; + weights.solution[stage] = problem_.stepper.solution_weights[stage]; weights.dense[stage] = rk_dense_weights_[stage]; } // The trial's own scalars are bound: their last row is the whole step's. - const StepExponent step{problem_.separable_geometry(), - spatial_, - active_mask_, - stage_phi_delta_, - phi_terms_, - problem_.n_pol_state, - problem_.n_stages, - problem_.n_phi_terms}; + const StepExponent step{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.dims.n_pol_state, + .n_stages = problem_.stepper.n_stages, + .n_phi = problem_.linear.n_phi_terms}; const auto current = current_; const auto stages = stages_; const auto coefficients = rk_dense_coefficients_; const auto stiff = rk_dense_stiff_; launch::parallel_for( "solprop_native.rk.dense_hold", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) SOLPROP_NATIVE_STRICT_FUNCTION { SOLPROP_NATIVE_STRICT_BLOCK @@ -9588,21 +9680,21 @@ void GeneralWorkspace::rk_dense_observe(const double theta, const TrialScalars& require_trial_scalars(carry); graph::deep_copy(execution_space_, rk_dense_held_, current_); upload_trial(carry); - const StepExponent flow{problem_.separable_geometry(), - spatial_, - active_mask_, - stage_phi_delta_, - phi_terms_, - problem_.n_pol_state, - problem_.n_stages, - problem_.n_phi_terms}; + const StepExponent flow{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.dims.n_pol_state, + .n_stages = problem_.stepper.n_stages, + .n_phi = problem_.linear.n_phi_terms}; const double rest = 1.0 - theta; const auto coefficients = rk_dense_coefficients_; const auto stiff = rk_dense_stiff_; const auto current = current_; launch::parallel_for( "solprop_native.rk.dense_observe", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) SOLPROP_NATIVE_STRICT_FUNCTION { SOLPROP_NATIVE_STRICT_BLOCK @@ -9644,7 +9736,8 @@ void GeneralWorkspace::rk_dense_release() noexcept { rk_dense_coefficients_ = {}; rk_dense_stiff_ = {}; rk_dense_held_ = {}; - rk_dense_weights_ = {}; + // A moved-in empty vector releases the storage and cannot throw (`= {}` may). + rk_dense_weights_ = std::vector(); rk_dense_ready_ = false; } @@ -9652,7 +9745,7 @@ std::vector GeneralWorkspace::collect_trial() { // A coupled flow's last stage is carried back by the step's own inverse action, so // it is not the next step's first stage; solprop's reference takes no FSAL there. pending_fsal_ = - problem_.first_same_as_last && !problem_.split_step_absorber && !flow_; + problem_.stepper.first_same_as_last && !problem_.absorber.split_step && !flow_; graph::deep_copy(execution_space_, pinned_partials_, partials_); execution_space_.fence("solprop_native trial boundary"); const auto& host = pinned_partials_; @@ -9671,9 +9764,9 @@ void GeneralWorkspace::reuse_fsal() { const auto solution = solution_; const auto error = error_; const auto whole = whole_propagator_; - const double solution_weight = problem_.solution_weights[0]; - const double error_weight = problem_.error_weights[0]; - const auto n = problem_.n_omega; + const double solution_weight = problem_.stepper.solution_weights[0]; + const double error_weight = problem_.stepper.error_weights[0]; + const auto n = problem_.dims.n_omega; launch::parallel_for( "solprop_native.stage.reuse_fsal", Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), @@ -9690,9 +9783,12 @@ GeneralWorkspace::transverse_chunk(const std::size_t first_unit) const noexcept const auto units = std::min(transverse_units_per_chunk(), transverse_units() - first_unit); if (problem_.slab_geometry()) - return {first_unit * problem_.n_p, units * problem_.n_p, first_unit * problem_.n_x, - units * problem_.n_x}; - return {first_unit, units, first_unit, units}; + return {.first_row = first_unit * problem_.spatial.n_p, + .rows = units * problem_.spatial.n_p, + .first_node = first_unit * problem_.spatial.n_x, + .nodes = units * problem_.spatial.n_x}; + return { + .first_row = first_unit, .rows = units, .first_node = first_unit, .nodes = units}; } void GeneralWorkspace::transverse_forward() { @@ -9728,8 +9824,8 @@ void GeneralWorkspace::separable_stage_input(const std::size_t stage, const doub const auto scratch = transverse_; const auto whole = whole_propagator_; const auto index = stage_index(); - const auto n_stages = problem_.n_stages; - const bool add_noise = problem_.modified_noise && has_noise_; + const auto n_stages = problem_.stepper.n_stages; + const bool add_noise = problem_.stepper.modified_noise && has_noise_; const double inverse_area_count = inverse_transverse_count(); // The stage input at one element, given its spatial value and its propagator. const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, @@ -9762,8 +9858,9 @@ void GeneralWorkspace::separable_stage_input(const std::size_t stage, const doub const bool last = stage + 1 == n_stages; crossing_tiles( "solprop_native.free.stage_input", execution_space_, problem_.n_columns(), - problem_.n_omega, [=](const CrossingTile& tile) { - PropagatorTile step, finish; + problem_.dims.n_omega, [=](const CrossingTile& tile) { + PropagatorTile step; + PropagatorTile finish; Complex linear[PropagatorTile::capacity]; tile.visit([&](const std::size_t column, const std::size_t omega) { const auto k = tile.at(column, omega); @@ -9792,7 +9889,7 @@ void GeneralWorkspace::separable_stage_input(const std::size_t stage, const doub } #endif crossing_for("solprop_native.free.stage_input", execution_space_, - problem_.n_columns(), problem_.n_omega, element); + problem_.n_columns(), problem_.dims.n_omega, element); } void GeneralWorkspace::separable_load_current() { @@ -9802,7 +9899,7 @@ void GeneralWorkspace::separable_load_current() { const double scale = inverse_transverse_count(); crossing_for( "solprop_native.free.current_transverse_input", execution_space_, - problem_.n_columns(), problem_.n_omega, + problem_.n_columns(), problem_.dims.n_omega, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { scratch.data()[index.cell(column, omega)] = scale * current(column, omega); }); @@ -9819,24 +9916,24 @@ void GeneralWorkspace::separable_pack_chunk(const TransverseChunk& chunk, const auto gaps = separable_gaps_; cartesian_for( "solprop_native.free.zero_fine_gaps", execution_space_, - chunk.rows * problem_.n_pol_state, gaps.extent(0), + chunk.rows * problem_.dims.n_pol_state, gaps.extent(0), KOKKOS_LAMBDA(const std::size_t row, const std::size_t gap) { fine(row, gaps(gap)) = Complex{}; }); } const auto scratch = transverse_; const auto crop = crop_index_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; [[maybe_unused]] const auto layout = transverse_layout(); const auto first = chunk.first_row; - const auto n = problem_.n_omega; - const auto n_to = problem_.n_to; - const bool analytic = problem_.family == FieldFamily::field_analytic; + const auto n = problem_.dims.n_omega; + const auto n_to = problem_.dims.n_to; + const bool analytic = problem_.dims.family == FieldFamily::field_analytic; // Free space's material field carries the pad scale here; the hybrid planar kind's // slab synthesis carries it (solprop's `Transform.synthesis`). - const double scale = - 1.0 / static_cast(material && !problem_.slab_geometry() ? problem_.n_t - : problem_.n_to); + const double scale = 1.0 / static_cast(material && !problem_.slab_geometry() + ? problem_.dims.n_t + : problem_.dims.n_to); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) tiled_crossing( "solprop_native.free.pack_temporal_chunk", execution_space_, chunk.rows * n_pol, @@ -9870,11 +9967,11 @@ void GeneralWorkspace::separable_pack_chunk(const TransverseChunk& chunk, // The chunk's physical nodes from its temporal rows: free space's are the rows // themselves; the hybrid planar kind's are the slab synthesis of each y row. void GeneralWorkspace::separable_nodes(const TransverseChunk& chunk) { - const bool real_field = problem_.family == FieldFamily::field_real; - const auto n_to = problem_.n_to; - const auto n_pol = problem_.n_pol_state; + const bool real_field = problem_.dims.family == FieldFamily::field_real; + const auto n_to = problem_.dims.n_to; + const auto n_pol = problem_.dims.n_pol_state; if (problem_.slab_geometry()) { - const auto units = chunk.nodes / problem_.n_x; + const auto units = chunk.nodes / problem_.spatial.n_x; if (real_field) slab_contract("solprop_native.slab.synthesis", execution_space_, synthesis_, modal_real_, node_real_, units, n_to); @@ -9903,12 +10000,12 @@ void GeneralWorkspace::separable_nodes(const TransverseChunk& chunk) { // columns: the polarisation, or one envelope-plasma current stream. void GeneralWorkspace::slab_project(const TransverseChunk& chunk, const EnvelopeStream* stream) { - const auto units = chunk.nodes / problem_.n_x; - const auto n_to = problem_.n_to; + const auto units = chunk.nodes / problem_.spatial.n_x; + const auto n_to = problem_.dims.n_to; if (stream != nullptr) slab_contract("solprop_native.slab.projection", execution_space_, projection_, stream->node, stream->projected, units, n_to); - else if (problem_.family == FieldFamily::envelope) + else if (problem_.dims.family == FieldFamily::envelope) slab_contract("solprop_native.slab.projection", execution_space_, projection_, polarisation_complex_, projected_complex_, units, n_to); else @@ -9949,7 +10046,7 @@ void GeneralWorkspace::separable_finish_transform(const std::size_t stage, // As the stage input: each tile's inverse propagators in one pass (issue #338). crossing_tiles( "solprop_native.free.finish_transform", execution_space_, problem_.n_columns(), - problem_.n_omega, [=](const CrossingTile& tile) { + problem_.dims.n_omega, [=](const CrossingTile& tile) { PropagatorTile inverse; Complex prefactor[PropagatorTile::capacity]; tile.visit([&](const std::size_t column, const std::size_t omega) { @@ -9972,7 +10069,7 @@ void GeneralWorkspace::separable_finish_transform(const std::size_t stage, #endif crossing_for("solprop_native.free.finish_transform", execution_space_, problem_.n_columns(), - problem_.n_omega, element); + problem_.dims.n_omega, element); } // Free space (solprop `NumpyRefBackend._nonlinear_free`) and the hybrid planar kind @@ -9985,12 +10082,12 @@ void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, const double dz, const bool accumulate) { auto* const dispersive = separable_dispersive_.get(); - const bool nonlinear = - problem_.nonlinear || problem_.plasma || problem_.raman || dispersive != nullptr; + const bool nonlinear = problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || dispersive != nullptr; const auto stages = stages_; const auto solution = solution_; const auto error = error_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; // A material finish assigns the stage row; only an absorber-only or a linear stage // accumulates into (or leaves) a cleared row. if (!nonlinear) @@ -10001,7 +10098,7 @@ void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, }); separable_stage_input(stage, dz); for (const bool material : {true, false}) { - if ((material && !nonlinear) || (!material && !problem_.in_stage_absorber)) + if ((material && !nonlinear) || (!material && !problem_.absorber.in_stage)) continue; if (!material) separable_stage_input(stage, dz); @@ -10010,8 +10107,8 @@ void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, } if (!accumulate) return; - const double weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; + const double weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; cartesian_for( "solprop_native.free.accumulate_stage", execution_space_, problem_.n_columns(), n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { @@ -10028,8 +10125,8 @@ void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, void GeneralWorkspace::separable_to_time(const TransverseChunk& chunk) { const bool tail = temporal_tail_rows_ != 0 && - chunk.rows * problem_.n_pol_state == temporal_tail_rows_; - if (problem_.family == FieldFamily::field_real) { + chunk.rows * problem_.dims.n_pol_state == temporal_tail_rows_; + if (problem_.dims.family == FieldFamily::field_real) { if (tail) temporal_tail_inverse_real_.execute(); else @@ -10046,8 +10143,8 @@ void GeneralWorkspace::separable_to_time(const TransverseChunk& chunk) { void GeneralWorkspace::separable_to_frequency(const TransverseChunk& chunk) { const bool tail = temporal_tail_rows_ != 0 && - chunk.rows * problem_.n_pol_state == temporal_tail_rows_; - if (problem_.family == FieldFamily::envelope) { + chunk.rows * problem_.dims.n_pol_state == temporal_tail_rows_; + if (problem_.dims.family == FieldFamily::envelope) { if (tail) temporal_tail_forward_complex_.execute(); else @@ -10074,13 +10171,13 @@ void GeneralWorkspace::separable_volume_core(const std::size_t stage) { graph::execute(*volume_inverse_, scratch, scratch, KokkosFFT::Normalization::none); const auto window = time_window_; const auto kerr_density = stage_kerr_density_; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double scale = 1.0 / static_cast(problem_.n_t); + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double scale = 1.0 / static_cast(problem_.dims.n_t); const auto plane = transverse_.extent(1) * transverse_.extent(2); const bool time_fastest = volume_order_; - const auto n_t = problem_.n_t; + const auto n_t = problem_.dims.n_t; launch::parallel_for( "solprop_native.free.volume_kerr", Kokkos::RangePolicy(execution_space_, 0, scratch.size()), @@ -10117,10 +10214,10 @@ void GeneralWorkspace::separable_spatial_core() { const auto index = register_index(); const auto spatial = spatial_; const auto scratch = transverse_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; cartesian_for( "solprop_native.free.absorber_spectrum", execution_space_, problem_.n_columns(), - problem_.n_omega, + problem_.dims.n_omega, KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { scratch.data()[index.cell(column, omega)] *= -0.5 * spatial.absorber(column / n_pol); @@ -10147,12 +10244,12 @@ void GeneralWorkspace::separable_core(const std::size_t stage, const bool materi // column, or alone in its place (`SeparableDispersive`). auto* const dispersive = separable_dispersive_.get(); const bool alone = dispersive != nullptr && dispersive->alone(); - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; // The host's scratch positions without a division (`RegisterIndex`). [[maybe_unused]] const auto index = register_index(); [[maybe_unused]] const auto layout = transverse_layout(); - const auto n = problem_.n_omega; - const auto n_to = problem_.n_to; + const auto n = problem_.dims.n_omega; + const auto n_to = problem_.dims.n_to; const auto real_time = modal_real_; const auto complex_time = modal_complex_; const auto polarisation_real = polarisation_real_; @@ -10164,8 +10261,8 @@ void GeneralWorkspace::separable_core(const std::size_t stage, const bool materi const auto spectrum = projected_spectrum_; const auto crop = crop_index_; const auto scratch = transverse_; - const bool real_field = problem_.family == FieldFamily::field_real; - const bool envelope = problem_.family == FieldFamily::envelope; + const bool real_field = problem_.dims.family == FieldFamily::field_real; + const bool envelope = problem_.dims.family == FieldFamily::envelope; const bool slab = problem_.slab_geometry(); // Profiling regions name the core's stages for a Kokkos Tools timer (SEP 28 phase // 0); with no tool loaded each is one test of a flag. The kernels inside carry @@ -10209,9 +10306,9 @@ void GeneralWorkspace::separable_core(const std::size_t stage, const bool materi Kokkos::Profiling::pushRegion("solprop_native.free.response"); if (!alone) response(stage, chunk.first_node, chunk.nodes); - if (problem_.envelope_plasma) + if (problem_.plasma.envelope) envelope_response(stage, chunk.first_node, chunk.nodes, &chunk); - if (problem_.raman) + if (problem_.raman.enabled) convolve_raman(stage, chunk.nodes); if (dispersive != nullptr) dispersive->respond(chunk.nodes); @@ -10261,7 +10358,7 @@ void GeneralWorkspace::separable_core(const std::size_t stage, const bool materi // The response's spectrum joins the chunk's, before the envelope currents'. if (responds) dispersive->combine(); - if (material && problem_.envelope_plasma) + if (material && problem_.plasma.envelope) envelope_transform(); if (material && held_plasma()) held_transform(); @@ -10297,7 +10394,7 @@ void GeneralWorkspace::separable_core(const std::size_t stage, const bool materi std::vector> GeneralWorkspace::fine_gap_runs() const { std::vector filled(fine_spectrum_size(), false); - for (const auto index : problem_.crop_index) + for (const auto index : problem_.grid.crop_index) filled[index] = true; std::vector> runs; for (std::size_t index = 0; index < filled.size(); ++index) { @@ -10349,21 +10446,21 @@ StageTerms GeneralWorkspace::stage_terms(const std::size_t stage) const { terms.gaps = fine_gaps_; terms.gap_ranges = fine_gaps_.extent(0) / 3; terms.active = active_mask_; - terms.n = problem_.n_omega; - terms.n_phi = problem_.n_phi_terms; - terms.n_stages = problem_.n_stages; - terms.n_prefactor = problem_.n_prefactor_terms; - terms.n_to = problem_.n_to; - terms.inverse_n_to = 1.0 / static_cast(problem_.n_to); - terms.general_pad = problem_.general_pad_scale; - terms.add_noise = problem_.modified_noise && has_noise_; - terms.analytic = problem_.family == FieldFamily::field_analytic; + terms.n = problem_.dims.n_omega; + terms.n_phi = problem_.linear.n_phi_terms; + terms.n_stages = problem_.stepper.n_stages; + terms.n_prefactor = problem_.linear.n_prefactor_terms; + terms.n_to = problem_.dims.n_to; + terms.inverse_n_to = 1.0 / static_cast(problem_.dims.n_to); + terms.general_pad = problem_.general.pad_scale; + terms.add_noise = problem_.stepper.modified_noise && has_noise_; + terms.analytic = problem_.dims.family == FieldFamily::field_analytic; const bool general_active = - terms.general && (general_->responds() || problem_.in_stage_absorber); - terms.zero_response = terms.general - ? !general_active - : !problem_.nonlinear && !problem_.plasma && - !problem_.raman && !absorbing() && !dispersive_; + terms.general && (general_->responds() || problem_.absorber.in_stage); + terms.zero_response = + terms.general ? !general_active + : !problem_.kerr.nonlinear && !problem_.plasma.enabled && + !problem_.raman.enabled && !absorbing() && !dispersive_; terms.in_stage_absorber = absorbing(); terms.identity_crop = identity_crop_; return terms; @@ -10397,14 +10494,14 @@ void GeneralWorkspace::stage_entry_with(const std::size_t stage, const double dz if (finish_previous && (stage == 0 || pending_stage_ + 1 != stage)) throw std::logic_error("a deferred stage finish lost its order"); const std::size_t previous = finish_previous ? stage - 1 : 0; - const double previous_solution = problem_.solution_weights[previous]; - const double previous_error = problem_.error_weights[previous]; - const double solution_weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; + const double previous_solution = problem_.stepper.solution_weights[previous]; + const double previous_error = problem_.stepper.error_weights[previous]; + const double solution_weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; const auto terms = stage_terms(stage); - const auto n = problem_.n_omega; - const auto n_phi = problem_.n_phi_terms; - const auto n_stages = problem_.n_stages; + const auto n = problem_.dims.n_omega; + const auto n_phi = problem_.linear.n_phi_terms; + const auto n_stages = problem_.stepper.n_stages; const auto gaps = problem_.separable_geometry() ? 0 : fine_spectrum_size() - n; const bool last = stage + 1 == n_stages; // The whole step's propagator is the last stage's when their phases are equal. @@ -10532,8 +10629,8 @@ void GeneralWorkspace::evaluate_stage(const std::size_t stage, const double dz) flow_->carry_input(stage, dz, current_, stages_, tableau_); evaluate_stage_source(stage, dz); flow_->add_residual(stage, stage_phase_); - flow_->carry_back(stage, problem_.solution_weights[stage], - problem_.error_weights[stage], stages_, solution_, error_); + flow_->carry_back(stage, problem_.stepper.solution_weights[stage], + problem_.stepper.error_weights[stage], stages_, solution_, error_); } void GeneralWorkspace::evaluate_stage_source(const std::size_t stage, const double dz, @@ -10556,13 +10653,13 @@ void GeneralWorkspace::evaluate_stage_source(const std::size_t stage, const doub // a response has none to finish. bool GeneralWorkspace::modal_core(const std::size_t stage) { if (general_) { - const bool general_active = general_->responds() || problem_.in_stage_absorber; + const bool general_active = general_->responds() || problem_.absorber.in_stage; if (general_active) general_core(stage); return general_active; } - if (!problem_.nonlinear && !problem_.plasma && !problem_.raman && !absorbing() && - !dispersive_) + if (!problem_.kerr.nonlinear && !problem_.plasma.enabled && !problem_.raman.enabled && + !absorbing() && !dispersive_) return false; // A device's forward rows go in one transform after the absorber's are formed. const bool packed = !forward_pack_.empty(); @@ -10572,7 +10669,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { dispersive_->prepare(stage); { const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.inverse"); - if (problem_.family == FieldFamily::field_real) + if (problem_.dims.family == FieldFamily::field_real) graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, KokkosFFT::Normalization::none); else if (!retained_synthesis_) @@ -10594,7 +10691,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { // a zero beta, the custom kernel by assignment, the folded column by its own // product), so only a stage that does not project needs them zeroed. if (!stage_projects) { - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { graph::deep_copy(execution_space_, projected_complex_, Complex{}); if (!projected_coarse_complex_.empty()) graph::deep_copy(execution_space_, projected_coarse_complex_, Complex{}); @@ -10610,17 +10707,17 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { convolve_raman_products(); bool first_chunk = true; std::size_t chunk = 0; - for (std::size_t first_node = 0; first_node < problem_.n_nodes; - first_node += problem_.node_chunk) { + for (std::size_t first_node = 0; first_node < problem_.dims.n_nodes; + first_node += problem_.dims.node_chunk) { const auto node_count = - std::min(problem_.node_chunk, problem_.n_nodes - first_node); + std::min(problem_.dims.node_chunk, problem_.dims.n_nodes - first_node); if (stage_synthesises) synthesise(first_node, node_count); if (!alone) response(stage, first_node, node_count); if (raman_products_ != 0) finish_raman_products(stage, first_node, node_count); - else if (problem_.raman) + else if (problem_.raman.enabled) convolve_raman(stage, node_count); if (dispersive_) dispersive_->respond(chunk, first_node, node_count); @@ -10634,7 +10731,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { const Kokkos::Profiling::ScopedRegion region("solprop_native.projection"); project(first_node, node_count, first_chunk); } - if (problem_.envelope_plasma) + if (problem_.plasma.envelope) envelope_response(stage, first_node, node_count); first_chunk = false; ++chunk; @@ -10643,7 +10740,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { dispersive_->finish(); if (!held_radial() && (!dispersive_ || !dispersive_->replaces_transform())) { const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward"); - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { graph::execute(*forward_complex_plan_, projected_complex_, projected_spectrum_, KokkosFFT::Normalization::none); if (forward_coarse_complex_plan_ && !defer_quadrature_) @@ -10663,7 +10760,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { if (absorbing()) { form_absorber(); const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.absorber"); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) graph::execute(*absorber_complex_plan_, absorber_complex_, absorber_spectrum_, KokkosFFT::Normalization::none); else if (!packed) @@ -10672,7 +10769,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { } if (packed) transform_forward_pack(); - if (problem_.envelope_plasma) + if (problem_.plasma.envelope) envelope_transform(); if (held_plasma() && !held_radial()) held_transform(false, packed); @@ -10686,7 +10783,7 @@ bool GeneralWorkspace::modal_core(const std::size_t stage) { // of the legacy node pipeline; the finish applies the shared normalisation and the // return to the interaction picture (solprop `NumpyRefBackend._nonlinear_general`). void GeneralWorkspace::general_core(const std::size_t stage) { - if (problem_.in_stage_absorber) { + if (problem_.absorber.in_stage) { graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, KokkosFFT::Normalization::none); form_absorber(); @@ -10700,8 +10797,9 @@ void GeneralWorkspace::general_core(const std::size_t stage) { if (general_->dispersive() != nullptr) general_->dispersive()->begin(stage); general_->evaluate(); - } else + } else { graph::deep_copy(execution_space_, general_->source(), Complex{}); + } } // The transforms scaled by stage `stage`'s taper, whose value `taper` the host holds. @@ -10725,11 +10823,11 @@ void GeneralWorkspace::scale_transform(const std::size_t stage, const auto scaled_projection_coarse_complex = scaled_projection_coarse_complex_; const auto scaled_projection_stacked_complex = scaled_projection_stacked_complex_; const auto taper = stage_taper_; - const auto rows = problem_.n_nodes * problem_.n_pol_transform; - const auto modes = problem_.n_modes; + const auto rows = problem_.dims.n_nodes * problem_.dims.n_pol_transform; + const auto modes = problem_.dims.n_modes; launch::parallel_for( "solprop_native.transform.scale", - Range2(execution_space_, {0, 0}, {rows, problem_.n_modes}), + Range2(execution_space_, {0, 0}, {rows, problem_.dims.n_modes}), KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { const double scale = taper(stage); scaled_synthesis(row, mode) = synthesis(row, mode) * scale; @@ -10765,8 +10863,8 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, synthesise_retained(first_node, node_count); return; } - const bool whole_transform = - first_node == 0 && node_count == problem_.n_nodes && problem_.n_pol_state == 1; + const bool whole_transform = first_node == 0 && node_count == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1; if (whole_transform && !radial_blas_input_.empty()) { packed_radial_gemm(scaled_synthesis_, modal_complex_, node_complex_); return; @@ -10776,8 +10874,9 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, // alias those rows directly instead of falling through to scalar synthesis. A // device takes the same GEMM wherever its shape guard admits the interleaved // synthesis, which `synthesis_provider` reports. - const bool radial_analytic_chunk = problem_.family == FieldFamily::field_analytic && - !problem_.spatial_spectral_weights.empty(); + const bool radial_analytic_chunk = + problem_.dims.family == FieldFamily::field_analytic && + !problem_.spatial.spectral_weights.empty(); if ((whole_transform || radial_analytic_chunk) && use_interleaved_synthesis()) { Kokkos::Profiling::pushRegion( "solprop_native.transform.synthesis.kokkosblas_interleaved"); @@ -10787,19 +10886,19 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, // twice as many real columns. static_assert(sizeof(Complex) == 2 * sizeof(double)); const Interleaved modal(reinterpret_cast(modal_complex_.data()), - problem_.n_modes, 2 * problem_.n_to); + problem_.dims.n_modes, 2 * problem_.dims.n_to); const Interleaved synthesis(scaled_synthesis_.data() + - first_node * problem_.n_modes, - node_count, problem_.n_modes); + first_node * problem_.dims.n_modes, + node_count, problem_.dims.n_modes); const Interleaved node(reinterpret_cast(node_complex_.data()), node_count, - 2 * problem_.n_to); + 2 * problem_.dims.n_to); engine_gemm(transform_space_for(node_count), synthesis, modal, node, 0.0); Kokkos::Profiling::popRegion(); return; } if (whole_transform && use_blas_for_synthesis()) { Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.kokkosblas"); - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) selected_gemm(transform_space_for(node_count), scaled_synthesis_complex_, modal_complex_, node_complex_, Complex{}, true); else if (one_mode_products_) @@ -10814,15 +10913,16 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, if (use_chunked_blas_for_synthesis()) { Kokkos::Profiling::pushRegion( "solprop_native.transform.synthesis.kokkosblas_chunked"); - const auto rows = node_count * problem_.n_pol_transform; - const auto first_row = first_node * problem_.n_pol_transform; - const bool tail = node_count != problem_.node_chunk; - if (problem_.family == FieldFamily::envelope) { + const auto rows = node_count * problem_.dims.n_pol_transform; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const bool tail = node_count != problem_.dims.node_chunk; + if (problem_.dims.family == FieldFamily::envelope) { const auto packed = tail ? tail_synthesis_complex_ : chunk_synthesis_complex_; const auto source = scaled_synthesis_complex_; launch::parallel_for( "solprop_native.transform.pack_synthesis_complex", - Range2(matrix_copy_space_for(node_count), {0, 0}, {rows, problem_.n_modes}), + Range2(matrix_copy_space_for(node_count), {0, 0}, + {rows, problem_.dims.n_modes}), KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { packed(row, mode) = source(first_row + row, mode); }); @@ -10835,7 +10935,8 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, const auto source = scaled_synthesis_; launch::parallel_for( "solprop_native.transform.pack_synthesis", - Range2(matrix_copy_space_for(node_count), {0, 0}, {rows, problem_.n_modes}), + Range2(matrix_copy_space_for(node_count), {0, 0}, + {rows, problem_.dims.n_modes}), KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { packed(row, mode) = source(first_row + row, mode); }); @@ -10853,11 +10954,11 @@ void GeneralWorkspace::synthesise(const std::size_t first_node, const auto modal_complex = modal_complex_; const auto node_real = node_real_; const auto node_complex = node_complex_; - const auto n_modes = problem_.n_modes; - const auto n_pol_state = problem_.n_pol_state; - const auto n_pol_transform = problem_.n_pol_transform; - const auto n_to = problem_.n_to; - const bool real_family = problem_.family == FieldFamily::field_real; + const auto n_modes = problem_.dims.n_modes; + const auto n_pol_state = problem_.dims.n_pol_state; + const auto n_pol_transform = problem_.dims.n_pol_transform; + const auto n_to = problem_.dims.n_to; + const bool real_family = problem_.dims.family == FieldFamily::field_real; const auto effective_rows = node_count * problem_.effective_node_pol(); launch::parallel_for( "solprop_native.transform.synthesis.custom", @@ -10890,7 +10991,7 @@ void GeneralWorkspace::response_harmonic(const std::size_t stage, const std::size_t first_node, const std::size_t node_count) { if constexpr (Nonlinear && Family != FieldFamily::field_real) { - if (problem_.harmonic_factor != 0.0) { + if (problem_.kerr.harmonic_factor != 0.0) { response_raman( stage, first_node, node_count); return; @@ -10905,8 +11006,8 @@ template (stage, first_node, node_count); } else { @@ -10939,7 +11040,7 @@ template void GeneralWorkspace::response_nonlinear(const std::size_t stage, const std::size_t first_node, const std::size_t node_count) { - if (problem_.nonlinear) { + if (problem_.kerr.nonlinear) { response_stash(stage, first_node, node_count); } else { response_stash(stage, first_node, node_count); @@ -10950,19 +11051,19 @@ template void GeneralWorkspace::response_plasma(const std::size_t stage, const std::size_t first_node, const std::size_t node_count) { - if (problem_.plasma && !problem_.envelope_plasma) { + if (problem_.plasma.enabled && !problem_.plasma.envelope) { if constexpr (Family != FieldFamily::envelope) { // SEP 27: the band column writes the physical current separately. The // ordinary traversal forms and collars only the Kerr/Raman polarisation. - if (problem_.plasma_band) { + if (problem_.plasma.band) { if (band_tiled_) { - if (problem_.n_channels > 1) + if (problem_.plasma.n_channels > 1) tiled_band_plasma_column(stage, first_node, node_count); else tiled_band_plasma_column(stage, first_node, node_count); - } else if (problem_.n_channels > 1) { + } else if (problem_.plasma.n_channels > 1) { team_band_plasma_column(stage, first_node, node_count); } else { @@ -10982,7 +11083,7 @@ void GeneralWorkspace::response_plasma(const std::size_t stage, void GeneralWorkspace::response(const std::size_t stage, const std::size_t first_node, const std::size_t node_count) { const auto components = problem_.effective_node_pol(); - switch (problem_.family) { + switch (problem_.dims.family) { case FieldFamily::field_real: if (components == 1) response_plasma(stage, first_node, node_count); @@ -11019,7 +11120,7 @@ void GeneralWorkspace::response_impl(const std::size_t stage, } } if constexpr (!Multiple) { - if (problem_.n_channels > 1 && Plasma) { + if (problem_.plasma.n_channels > 1 && Plasma) { response_impl(stage, first_node, node_count); return; @@ -11040,28 +11141,28 @@ void GeneralWorkspace::response_impl(const std::size_t stage, const auto stash = stash_candidate_; const auto density = stage_plasma_density_; const auto kerr_density = stage_kerr_density_; - const auto channels = problem_.n_channels; + const auto channels = problem_.plasma.n_channels; const auto channel_parameters = channel_parameters_; const auto channel_coefficients = channel_coefficients_; const auto channel_density = stage_channel_density_; const auto channel_preionisation = stage_channel_preionisation_; const auto scratch = channel_scratch_; const auto preionisation = stage_preionisation_; - const auto n_to = problem_.n_to; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double harmonic_factor = problem_.harmonic_factor; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const auto n_stages = problem_.n_stages; + const auto n_to = problem_.dims.n_to; + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double harmonic_factor = problem_.kerr.harmonic_factor; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const auto n_stages = problem_.stepper.n_stages; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // The device's form (`device_response`); a mixture of channels takes the generic // column below. @@ -11119,11 +11220,11 @@ void GeneralWorkspace::response_impl(const std::size_t stage, // Cache contiguous pointwise addresses to avoid repeated View stride loads. // Plasma retains its measured recurrence-loop layout; applying // the same change there regresses HE plasma by 8-9% (kokkos-reopt report). - const auto real_input = input_real.data(); - const auto complex_input = input_complex.data(); - const auto real_target = target_real.data(); - const auto complex_target = target_complex.data(); - const auto window_data = window.data(); + auto* const real_input = input_real.data(); + auto* const complex_input = input_complex.data(); + auto* const real_target = target_real.data(); + auto* const complex_target = target_complex.data(); + auto* const window_data = window.data(); const double pointwise_factor = Plasma ? 0.0 : kerr_coefficient * kerr_density(stage); for (std::size_t time = 0; time < n_to; ++time) { @@ -11328,11 +11429,11 @@ void GeneralWorkspace::device_response(const std::size_t stage, const auto window = time_window_; const auto carrier = carrier_; const auto kerr_density = stage_kerr_density_; - const auto n_to = problem_.n_to; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double harmonic_factor = problem_.harmonic_factor; + const auto n_to = problem_.dims.n_to; + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double harmonic_factor = problem_.kerr.harmonic_factor; if constexpr (Plasma) device_trapezoid_plasma(stage, first_node, node_count); launch::parallel_for( @@ -11425,21 +11526,22 @@ void GeneralWorkspace::device_trapezoid_plasma(const std::size_t stage, const auto rate_coefficients = rate_coefficients_; const auto density = stage_plasma_density_; const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const auto n_to = problem_.n_to; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const auto n_to = problem_.dims.n_to; const auto samples = static_cast(n_to); const auto width = static_cast(plasma_tile_samples); const auto tiles = static_cast(plasma_tiles_); const auto last = n_to - 1; - const bool stashing = Stash && !stash.empty() && stage + 1 == problem_.n_stages; + const bool stashing = + Stash && !stash.empty() && stage + 1 == problem_.stepper.n_stages; const std::size_t rows = node_count * Components; const auto team_size = static_cast(response_team_size()); // The field a row reads: a real row's value, or an analytic or envelope row's real @@ -11629,7 +11731,7 @@ KOKKOS_INLINE_FUNCTION BandWindow band_window_of(const BandLive& live, if constexpr (band_windows) return band_support(live.min_val, live.max_val, samples, half); else - return {0, samples - 1}; + return {.first = 0, .last = samples - 1}; } /** Count `index` into `live` when `value` is not exactly zero. */ @@ -11726,17 +11828,13 @@ int GeneralWorkspace::band_host_team_size(const std::size_t node_count) const { const auto per_node = std::max(1, threads / std::max(1, node_count)); const auto by_samples = - std::max(1, problem_.n_to / band_host_samples_per_thread); + std::max(1, problem_.dims.n_to / band_host_samples_per_thread); return static_cast(std::min(per_node, by_samples)); #endif } std::size_t GeneralWorkspace::band_league(const std::size_t node_count) const { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return node_count; -#else - return std::min(node_count, band_slots_); -#endif + return build::device ? node_count : std::min(node_count, band_slots_); } template @@ -11746,7 +11844,7 @@ GeneralWorkspace::band_team_policy(const std::size_t node_count, using Policy = Kokkos::TeamPolicy; constexpr auto policy_limit = static_cast(std::numeric_limits::max()); - if (node_count > policy_limit || problem_.n_to > policy_limit) + if (node_count > policy_limit || problem_.dims.n_to > policy_limit) throw std::length_error("band team column extent exceeds signed 32-bit range"); const auto league = static_cast(band_league(node_count)); const auto scratch = @@ -11812,7 +11910,7 @@ void GeneralWorkspace::team_band_plasma_column(const std::size_t stage, const auto integrands = plasma_current_integrand_; const auto taps_view = plasma_taps_; const auto stash = stash_candidate_; - const bool stashing = !stash.empty() && stage + 1 == problem_.n_stages; + const bool stashing = !stash.empty() && stage + 1 == problem_.stepper.n_stages; const auto rate_coefficients = rate_coefficients_; const auto channel_parameters = channel_parameters_; const auto channel_coefficients = channel_coefficients_; @@ -11820,22 +11918,22 @@ void GeneralWorkspace::team_band_plasma_column(const std::size_t stage, const auto channel_preionisation = stage_channel_preionisation_; const auto density = stage_plasma_density_; const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const std::size_t channels = Multiple ? problem_.n_channels : 1; - const auto samples = static_cast(problem_.n_to); + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const std::size_t channels = Multiple ? problem_.plasma.n_channels : 1; + const auto samples = static_cast(problem_.dims.n_to); #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - const std::size_t sample_count = problem_.n_to; + const std::size_t sample_count = problem_.dims.n_to; #endif - const auto half = static_cast(problem_.plasma_half_width()); - const std::size_t last = problem_.n_to - 1; + const auto half = static_cast(problem_.plasma.half_width()); + const std::size_t last = problem_.dims.n_to - 1; const std::size_t league = band_league(node_count); const auto column = KOKKOS_LAMBDA(const TeamMember& team) { // CUDA extended-lambda restriction 12 requires the first capture outside @@ -12230,7 +12328,7 @@ GeneralWorkspace::tiled_band_policy(const std::size_t rows, const bool scratch, constexpr auto policy_limit = static_cast(std::numeric_limits::max()); if (band_tile_count_ == 0 || rows > policy_limit / band_tile_count_ || - problem_.n_to > policy_limit) + problem_.dims.n_to > policy_limit) throw std::length_error("tiled band column extent exceeds signed 32-bit range"); const auto league = static_cast(rows * band_tile_count_); const auto bytes = scratch ? band_tile_bytes(band_tile_) : 0; @@ -12284,10 +12382,15 @@ void GeneralWorkspace::tiled_band_plasma_column(const std::size_t stage, constexpr bool folded = Folded; const BandField field{folded ? modal_real_ : node_real_, folded ? modal_complex_ : node_complex_}; - const BandRate rate{ - rate_coefficients_, channel_parameters_, channel_coefficients_, - problem_.log_field_min, problem_.d_log_field, problem_.field_min, - problem_.field_max, problem_.rate_max, problem_.rate_intervals}; + const BandRate rate{rate_coefficients_, + channel_parameters_, + channel_coefficients_, + problem_.plasma.log_field_min, + problem_.plasma.d_log_field, + problem_.plasma.field_min, + problem_.plasma.field_max, + problem_.plasma.rate_max, + problem_.plasma.rate_intervals}; const auto synthesis = synthesis_; const auto taper = stage_taper_; const auto out = held_current_; @@ -12300,18 +12403,18 @@ void GeneralWorkspace::tiled_band_plasma_column(const std::size_t stage, const auto totals = band_tile_totals_; const auto taps_view = plasma_taps_; const auto stash = stash_candidate_; - const bool stashing = !stash.empty() && stage + 1 == problem_.n_stages; + const bool stashing = !stash.empty() && stage + 1 == problem_.stepper.n_stages; const auto channel_parameters = channel_parameters_; const auto channel_density = stage_channel_density_; const auto channel_preionisation = stage_channel_preionisation_; const auto density = stage_plasma_density_; const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const std::size_t channels = Multiple ? problem_.n_channels : 1; - const auto samples = static_cast(problem_.n_to); - const auto half = static_cast(problem_.plasma_half_width()); + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const std::size_t channels = Multiple ? problem_.plasma.n_channels : 1; + const auto samples = static_cast(problem_.dims.n_to); + const auto half = static_cast(problem_.plasma.half_width()); const auto width = static_cast(band_tile_); const auto tiles = static_cast(band_tile_count_); const int last = samples - 1; @@ -12463,9 +12566,9 @@ void GeneralWorkspace::tiled_band_local(const RealView2& source, const std::size const auto locals = plasma_outer_current_; const auto totals = band_tile_totals_; const auto taps_view = plasma_taps_; - const double dt = problem_.dt_fine; - const auto samples = static_cast(problem_.n_to); - const auto half = static_cast(problem_.plasma_half_width()); + const double dt = problem_.grid.dt_fine; + const auto samples = static_cast(problem_.dims.n_to); + const auto half = static_cast(problem_.plasma.half_width()); const auto width = static_cast(band_tile_); const auto tiles = static_cast(band_tile_count_); const auto local = KOKKOS_LAMBDA(const TeamMember& team) { @@ -12508,7 +12611,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, using TeamMember = Kokkos::TeamPolicy::member_type; const auto node_real = node_real_; const auto node_complex = node_complex_; - const bool real_family = problem_.family == FieldFamily::field_real; + const bool real_family = problem_.dims.family == FieldFamily::field_real; const auto rates = plasma_rate_; const auto filtered = plasma_fraction_; const auto exposures = plasma_exposure_; @@ -12518,20 +12621,20 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, const auto parameters = channel_parameters_; const auto channel_coefficients = channel_coefficients_; const auto channel_preionisation = stage_channel_preionisation_; - const auto channels = problem_.n_channels; + const auto channels = problem_.plasma.n_channels; const bool multiple = channels > 1; const auto n_pol = problem_.effective_node_pol(); - const auto samples = static_cast(problem_.n_to); - const auto intervals = problem_.rate_intervals; - const auto dt = problem_.dt_fine; - const auto log_field_min = problem_.log_field_min; - const auto d_log_field = problem_.d_log_field; - const auto field_min = problem_.field_min; - const auto field_max = problem_.field_max; - const auto rate_max = problem_.rate_max; - const auto half = static_cast(problem_.plasma_half_width()); + const auto samples = static_cast(problem_.dims.n_to); + const auto intervals = problem_.plasma.rate_intervals; + const auto dt = problem_.grid.dt_fine; + const auto log_field_min = problem_.plasma.log_field_min; + const auto d_log_field = problem_.plasma.d_log_field; + const auto field_min = problem_.plasma.field_min; + const auto field_max = problem_.plasma.field_max; + const auto rate_max = problem_.plasma.rate_max; + const auto half = static_cast(problem_.plasma.half_width()); const int reach = Kokkos::min(2 * half, samples); - const std::size_t last = problem_.n_to - 1; + const std::size_t last = problem_.dims.n_to - 1; const std::size_t league = band_league(node_count); const auto scan = KOKKOS_LAMBDA(const TeamMember& team) { capture_anchor(node_real, node_complex, real_family, rates, filtered, exposures, @@ -12588,7 +12691,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, band_rates_block(magnitudes, &rates(slot, start), count, lower, upper, highest, within); }); - } else + } else { #endif Kokkos::parallel_for( Kokkos::TeamThreadRange(team, samples), [&](const int sample) { @@ -12609,6 +12712,9 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, d_log_field, field_min, field_max, rate_max, intervals); }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif team.team_barrier(); team_band_filter(team, tile, rates, slot, filtered, slot, samples, taps, half); @@ -12683,7 +12789,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, exposed[k] = fraction[k]; } }); - } else + } else { #endif Kokkos::parallel_for( Kokkos::TeamThreadRange(team, samples), [&](const int sample) { @@ -12692,6 +12798,9 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, filtered(slot, time) = rates(slot, time) * (1.0 - fraction); exposures(slot, time) = fraction; }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif team.team_barrier(); double paid = 0.0; Kokkos::parallel_reduce( @@ -12737,7 +12846,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, void GeneralWorkspace::convolve_raman(const std::size_t stage, const std::size_t node_count) { - const bool tail = node_count != problem_.node_chunk; + const bool tail = node_count != problem_.dims.node_chunk; Kokkos::Profiling::pushRegion("solprop_native.raman.drive_fft"); execute_raman(tail ? *raman_tail_forward_plan_ : *raman_drive_forward_plan_, tail ? raman_tail_drive_ : node_raman_drive_, @@ -12749,11 +12858,11 @@ void GeneralWorkspace::convolve_raman(const std::size_t stage, const auto node_spectrum = node_raman_spectrum_; const auto kernel_spectrum = raman_kernel_spectrum_; const auto drives = problem_.raman_drives(); - const double inverse_doubled = 1.0 / static_cast(2 * problem_.n_to); + const double inverse_doubled = 1.0 / static_cast(2 * problem_.dims.n_to); launch::parallel_for( "solprop_native.raman.spectral_multiply", Range2(tail ? raman_tail_multiply_space_ : raman_multiply_space_, {0, 0}, - {node_count * drives, problem_.n_to + 1}), + {node_count * drives, problem_.dims.n_to + 1}), KOKKOS_LAMBDA(const std::size_t node, const std::size_t index) { node_spectrum(node, index) = (node_spectrum(node, index) * @@ -12781,13 +12890,13 @@ void GeneralWorkspace::finish_raman(const std::size_t stage, const auto convolved = node_raman_convolved_; const auto window = time_window_; const auto density = stage_raman_density_; - const auto family = problem_.family; + const auto family = problem_.dims.family; const auto components = problem_.effective_node_pol(); const auto drives = problem_.raman_drives(); - const double epsilon0 = problem_.raman_epsilon0; + const double epsilon0 = problem_.raman.epsilon0; launch::parallel_for( "solprop_native.raman.finish", - Range2(response_space_for(node_count), {0, 0}, {node_count, problem_.n_to}), + Range2(response_space_for(node_count), {0, 0}, {node_count, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { if (components == 2) { const auto base = 2 * node; @@ -12797,8 +12906,8 @@ void GeneralWorkspace::finish_raman(const std::size_t stage, const double w = drives == 3 ? convolved(row + 2, time) : 0.0; const double factor = epsilon0 * density(stage); if (family == FieldFamily::envelope) { - const Complex x = node_complex(base, time), - y = node_complex(base + 1, time); + const Complex x = node_complex(base, time); + const Complex y = node_complex(base + 1, time); out_complex(base, time) = (out_complex(base, time) + factor * ((u + v) * x + w * y)) * window(time); @@ -12850,13 +12959,16 @@ std::size_t GeneralWorkspace::raman_modal_products() const noexcept { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) return 0; #else - if (!problem_.raman || problem_.general_modal || problem_.dispersive || - problem_.separable_geometry() || problem_.effective_node_pol() != 1 || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1 || - problem_.raman_drives() != 1 || problem_.n_nodes < 2) + if (!problem_.raman.enabled || problem_.general.enabled || + problem_.dispersive.enabled || problem_.separable_geometry() || + problem_.effective_node_pol() != 1 || problem_.dims.n_pol_state != 1 || + problem_.dims.n_pol_transform != 1 || problem_.raman_drives() != 1 || + problem_.dims.n_nodes < 2) return 0; - const auto products = problem_.n_modes * (problem_.n_modes + 1) / 2; - return products < problem_.n_nodes && products <= problem_.node_chunk ? products : 0; + const auto products = problem_.dims.n_modes * (problem_.dims.n_modes + 1) / 2; + return products < problem_.dims.n_nodes && products <= problem_.dims.node_chunk + ? products + : 0; #endif } @@ -12865,10 +12977,10 @@ void GeneralWorkspace::convolve_raman_products() { const auto drive = raman_product_drive_; const auto modal_real = modal_real_; const auto modal_complex = modal_complex_; - const auto modes = problem_.n_modes; - const auto n_to = problem_.n_to; - const bool real_family = problem_.family == FieldFamily::field_real; - const bool squared = problem_.raman_thg; + const auto modes = problem_.dims.n_modes; + const auto n_to = problem_.dims.n_to; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const bool squared = problem_.raman.thg; launch::parallel_for( "solprop_native.raman.products", Range2(raman_product_space_, {0, 0}, {raman_products_, 2 * n_to}), @@ -12930,10 +13042,10 @@ void GeneralWorkspace::finish_raman_products(const std::size_t stage, const auto synthesis = scaled_synthesis_; const auto window = time_window_; const auto density = stage_raman_density_; - const auto family = problem_.family; - const auto modes = problem_.n_modes; - const double epsilon0 = problem_.raman_epsilon0; - const auto n_to = problem_.n_to; + const auto family = problem_.dims.family; + const auto modes = problem_.dims.n_modes; + const double epsilon0 = problem_.raman.epsilon0; + const auto n_to = problem_.dims.n_to; // Blocks of samples per node, the products accumulated in their order over each // block as contiguous rows: one pass over the block per product, which vectorises, // where a per-sample loop over the products strides between rows. @@ -13003,7 +13115,7 @@ void GeneralWorkspace::envelope_cycle(const std::size_t channel, const auto moments = envelope_moments_; const auto rates = envelope_rates_; const auto failures = envelope_failures_; - const auto n_to = problem_.n_to; + const auto n_to = problem_.dims.n_to; launch::parallel_for( "solprop_native.envelope.cycle_moments", Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), @@ -13071,14 +13183,13 @@ void GeneralWorkspace::envelope_cycle(const std::size_t channel, envelope_status_); response_space_for(node_count).fence("solprop_native envelope refinement boundary"); if (envelope_host_status_(0) != no_failure) - throw EnvelopeConvergenceError("envelope plasma channel " + - std::to_string(channel) + - " exceeded the angular refinement budget"); + refuse("envelope plasma channel ", channel, + " exceeded the angular refinement budget"); const auto rows = std::make_pair(std::size_t{0}, node_count); portable_team_trapezoid_scan( response_space_for(node_count), Kokkos::subview(envelope_rates_, rows, Kokkos::ALL()), - Kokkos::subview(envelope_exposure_, rows, Kokkos::ALL()), problem_.dt_fine); + Kokkos::subview(envelope_exposure_, rows, Kokkos::ALL()), problem_.grid.dt_fine); } // One channel's exposure and peak cycle-averaged rate at each node of `field` @@ -13092,7 +13203,7 @@ void GeneralWorkspace::envelope_statistics(const std::size_t channel, envelope_cycle(channel, node_count, field, n_pol); const auto rates = envelope_rates_; const auto exposure = envelope_exposure_; - const auto n_to = problem_.n_to; + const auto n_to = problem_.dims.n_to; launch::parallel_for( "solprop_native.envelope.population_statistics", Kokkos::RangePolicy(response_space_for(node_count), 0, @@ -13112,7 +13223,7 @@ void GeneralWorkspace::envelope_population(const std::size_t channel, const auto n_pol = problem_.effective_node_pol(); envelope_cycle(channel, node_count, node_complex_, n_pol); const auto exposure = envelope_exposure_; - const auto n_to = problem_.n_to; + const auto n_to = problem_.dims.n_to; const auto parameters = channel_parameters_; const auto densities = stage_channel_density_; const auto initials = stage_channel_preionisation_; @@ -13155,7 +13266,7 @@ void GeneralWorkspace::envelope_response(const std::size_t stage, const std::size_t first_node, const std::size_t node_count, const TransverseChunk* chunk) { - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) envelope_population(channel, stage, node_count); for (auto& stream : envelope_streams_) { if (problem_.slab_geometry()) { @@ -13190,7 +13301,7 @@ void GeneralWorkspace::held_response(const std::size_t stage, const auto product = held_streams_[0].node; const auto edge = held_streams_[1].node; const auto live = node_count * problem_.effective_node_pol(); - const auto n_to = problem_.n_to; + const auto n_to = problem_.dims.n_to; cartesian_for( "solprop_native.held.products", response_space_for(node_count), current.extent(0), n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { @@ -13216,7 +13327,7 @@ void GeneralWorkspace::held_response(const std::size_t stage, graph::deep_copy(response_space_for(node_count), stream.projected, 0.0); slab_contract("solprop_native.held.slab_projection", response_space_for(node_count), projection_, stream.node, - stream.projected, chunk->nodes / problem_.n_x, n_to); + stream.projected, chunk->nodes / problem_.spatial.n_x, n_to); } else if (problem_.free_geometry() || radial) { graph::deep_copy(response_space_for(node_count), stream.projected, stream.node); } else { @@ -13249,16 +13360,16 @@ void GeneralWorkspace::gated_response(const std::size_t node_count) { const auto current = Kokkos::subview(held_current_, live, Kokkos::ALL); const auto prefix = Kokkos::subview(gated_prefix_, live, Kokkos::ALL); const auto& space = response_space_for(node_count); - if (problem_.plasma_band) + if (problem_.plasma.band) portable_team_band_scan(space, current, Kokkos::subview(gated_filtered_, live, Kokkos::ALL), prefix, - plasma_taps_, problem_.dt_fine); + plasma_taps_, problem_.grid.dt_fine); else - portable_team_trapezoid_scan(space, current, prefix, problem_.dt_fine); + portable_team_trapezoid_scan(space, current, prefix, problem_.grid.dt_fine); const auto window = time_window_; const auto polarisation = polarisation_real_; cartesian_for( - "solprop_native.gated.polarisation", space, live.second, problem_.n_to, + "solprop_native.gated.polarisation", space, live.second, problem_.dims.n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { polarisation(row, time) += window(time) * prefix(row, time); }); @@ -13324,7 +13435,8 @@ void GeneralWorkspace::held_transform(const bool coarse_only, const bool transfo const bool include_coarse = !coarse.empty() && (coarse_only || !defer_quadrature_); cartesian_for( "solprop_native.held.spectral_assembly", execution_space_, projected.extent(0), - problem_.n_omega, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { const auto bin = crop(omega); if (active(omega)) { const double value = inverse(omega); @@ -13360,7 +13472,7 @@ void GeneralWorkspace::held_radial_projection(const std::size_t first_node, const auto inverse = held_inverse_; const auto active = active_mask_; const auto crop = crop_index_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const auto live = node_count * problem_.effective_node_pol(); cartesian_for( "solprop_native.held.radial_assembly", node_spectral_space_, nodal.extent(0), n, @@ -13376,8 +13488,8 @@ void GeneralWorkspace::held_radial_projection(const std::size_t first_node, }); const auto packed = held_projection_; const auto projection = scaled_projection_; - const auto first_row = first_node * problem_.n_pol_transform; - const auto live_rows = node_count * problem_.n_pol_transform; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const auto live_rows = node_count * problem_.dims.n_pol_transform; cartesian_for( "solprop_native.held.radial_projection_matrix", matrix_copy_space_, packed.extent(0), packed.extent(1), @@ -13389,11 +13501,11 @@ void GeneralWorkspace::held_radial_projection(const std::size_t first_node, using Interleaved = Kokkos::View>; static_assert(sizeof(Complex) == 2 * sizeof(double)); - const auto columns = 2 * problem_.n_pol_state * n; + const auto columns = 2 * problem_.dims.n_pol_state * n; const Interleaved input(reinterpret_cast(nodal.data()), packed.extent(1), columns); const Interleaved output(reinterpret_cast(held_projected_spectrum_.data()), - problem_.n_modes, columns); + problem_.dims.n_modes, columns); selected_gemm(transform_space_for(node_count), packed, input, output, 0.0, true); if (first_node == 0) graph::deep_copy(temporal_execution_space_, projected_spectrum_, Complex{}); @@ -13426,7 +13538,7 @@ void GeneralWorkspace::envelope_transform() { const auto crop = crop_index_; launch::parallel_for( "solprop_native.envelope.spectral_assembly", - Range2(execution_space_, {0, 0}, {projected.extent(0), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {projected.extent(0), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { const auto index = crop(omega); projected(column, index) += drude_factor(omega) * drive(column, index) + @@ -13456,8 +13568,8 @@ void GeneralWorkspace::project(const std::size_t first_node, stream == nullptr ? projected_coarse_complex_ : stream->coarse; const auto stacked_target = stream == nullptr ? projected_stacked_complex_ : stream->stacked; - const bool whole_transform = - first_node == 0 && node_count == problem_.n_nodes && problem_.n_pol_state == 1; + const bool whole_transform = first_node == 0 && node_count == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1; if (whole_transform && !radial_blas_input_.empty()) { packed_radial_gemm(scaled_projection_, node_source, projected_target); return; @@ -13469,13 +13581,13 @@ void GeneralWorkspace::project(const std::size_t first_node, Kokkos::MemoryTraits>; static_assert(sizeof(Complex) == 2 * sizeof(double)); const Interleaved nodes(reinterpret_cast(node_source.data()), - problem_.n_nodes, 2 * problem_.n_to); + problem_.dims.n_nodes, 2 * problem_.dims.n_to); const Interleaved fine(reinterpret_cast(projected_target.data()), - problem_.n_modes, 2 * problem_.n_to); + problem_.dims.n_modes, 2 * problem_.dims.n_to); engine_gemm(transform_space_for(node_count), scaled_projection_, nodes, fine, 0.0); - if (project_coarse_ && !problem_.projection_coarse_matrix.empty()) { + if (project_coarse_ && !problem_.transform.projection_coarse_matrix.empty()) { const Interleaved coarse(reinterpret_cast(coarse_target.data()), - problem_.n_modes, 2 * problem_.n_to); + problem_.dims.n_modes, 2 * problem_.dims.n_to); engine_gemm(transform_space_for(node_count), scaled_projection_coarse_, nodes, coarse, 0.0); } @@ -13488,7 +13600,7 @@ void GeneralWorkspace::project(const std::size_t first_node, // splits"). if (whole_transform && use_blas_for_projection()) { Kokkos::Profiling::pushRegion("solprop_native.transform.projection.kokkosblas"); - if (problem_.family == FieldFamily::envelope && project_coarse_ && + if (problem_.dims.family == FieldFamily::envelope && project_coarse_ && !scaled_projection_stacked_complex_.empty()) { selected_gemm(transform_space_for(node_count), scaled_projection_stacked_complex_, node_source, stacked_target, Complex{}, true); @@ -13498,12 +13610,12 @@ void GeneralWorkspace::project(const std::size_t first_node, const auto columns = problem_.n_columns(); launch::parallel_for( "solprop_native.transform.projection.split_stacked_complex", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { fine(column, time) = stacked(column, time); coarse(column, time) = stacked(columns + column, time); }); - } else if (problem_.family == FieldFamily::envelope) { + } else if (problem_.dims.family == FieldFamily::envelope) { selected_gemm(transform_space_for(node_count), scaled_projection_complex_, node_source, projected_target, Complex{}, true); } else if (one_mode_products_) { @@ -13520,7 +13632,7 @@ void GeneralWorkspace::project(const std::size_t first_node, const auto columns = problem_.n_columns(); launch::parallel_for( "solprop_native.transform.projection.split_stacked", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { fine(column, time) = stacked(column, time); coarse(column, time) = stacked(columns + column, time); @@ -13535,12 +13647,13 @@ void GeneralWorkspace::project(const std::size_t first_node, if (use_chunked_blas_for_projection()) { Kokkos::Profiling::pushRegion( "solprop_native.transform.projection.kokkosblas_chunked"); - const auto rows = node_count * problem_.n_pol_transform; - const auto first_row = first_node * problem_.n_pol_transform; - const bool tail = node_count != problem_.node_chunk; - const bool last_chunk = first_node + node_count == problem_.n_nodes; - const bool stacked = project_coarse_ && !problem_.projection_coarse_matrix.empty(); - if (problem_.family == FieldFamily::envelope) { + const auto rows = node_count * problem_.dims.n_pol_transform; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const bool tail = node_count != problem_.dims.node_chunk; + const bool last_chunk = first_node + node_count == problem_.dims.n_nodes; + const bool stacked = + project_coarse_ && !problem_.transform.projection_coarse_matrix.empty(); + if (problem_.dims.family == FieldFamily::envelope) { const auto polarisation = Kokkos::subview( node_source, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); if (stacked) { @@ -13550,7 +13663,7 @@ void GeneralWorkspace::project(const std::size_t first_node, launch::parallel_for( "solprop_native.transform.pack_projection_stacked_complex", Range2(matrix_copy_space_for(node_count, true), {0, 0}, - {2 * problem_.n_modes, rows}), + {2 * problem_.dims.n_modes, rows}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { packed(mode, row) = source(mode, first_row + row); }); @@ -13564,7 +13677,7 @@ void GeneralWorkspace::project(const std::size_t first_node, const auto columns = problem_.n_columns(); launch::parallel_for( "solprop_native.transform.projection.split_stacked_complex", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { fine(column, time) = projected(column, time); coarse(column, time) = projected(columns + column, time); @@ -13579,12 +13692,13 @@ void GeneralWorkspace::project(const std::size_t first_node, scratch.empty() ? (tail ? tail_projection_complex_ : chunk_projection_complex_) : ComplexView2(Kokkos::subview( - scratch, std::pair{std::size_t{0}, problem_.n_modes}, + scratch, std::pair{std::size_t{0}, problem_.dims.n_modes}, Kokkos::ALL())); const auto source = scaled_projection_complex_; launch::parallel_for( "solprop_native.transform.pack_projection_complex", - Range2(matrix_copy_space_for(node_count), {0, 0}, {problem_.n_modes, rows}), + Range2(matrix_copy_space_for(node_count), {0, 0}, + {problem_.dims.n_modes, rows}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { packed(mode, row) = source(mode, first_row + row); }); @@ -13601,7 +13715,7 @@ void GeneralWorkspace::project(const std::size_t first_node, launch::parallel_for( "solprop_native.transform.pack_projection_stacked", Range2(matrix_copy_space_for(node_count, true), {0, 0}, - {2 * problem_.n_modes, rows}), + {2 * problem_.dims.n_modes, rows}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { packed(mode, row) = source(mode, first_row + row); }); @@ -13614,7 +13728,7 @@ void GeneralWorkspace::project(const std::size_t first_node, const auto columns = problem_.n_columns(); launch::parallel_for( "solprop_native.transform.projection.split_stacked", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { fine(column, time) = projected(column, time); coarse(column, time) = projected(columns + column, time); @@ -13626,13 +13740,14 @@ void GeneralWorkspace::project(const std::size_t first_node, const RealView2 packed = scratch.empty() ? (tail ? tail_projection_ : chunk_projection_) - : RealView2(Kokkos::subview(scratch, - std::pair{std::size_t{0}, problem_.n_modes}, - Kokkos::ALL())); + : RealView2(Kokkos::subview( + scratch, std::pair{std::size_t{0}, problem_.dims.n_modes}, + Kokkos::ALL())); const auto source = scaled_projection_; launch::parallel_for( "solprop_native.transform.pack_projection", - Range2(matrix_copy_space_for(node_count), {0, 0}, {problem_.n_modes, rows}), + Range2(matrix_copy_space_for(node_count), {0, 0}, + {problem_.dims.n_modes, rows}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { packed(mode, row) = source(mode, first_row + row); }); @@ -13652,10 +13767,10 @@ void GeneralWorkspace::project(const std::size_t first_node, const auto& projected_complex = projected_target; const auto& projected_coarse_real = real_coarse_target; const auto& projected_coarse_complex = coarse_target; - const auto n_pol_state = problem_.n_pol_state; - const auto n_pol_transform = problem_.n_pol_transform; - const auto n_to = problem_.n_to; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n_pol_state = problem_.dims.n_pol_state; + const auto n_pol_transform = problem_.dims.n_pol_transform; + const auto n_to = problem_.dims.n_to; + const bool envelope = problem_.dims.family == FieldFamily::envelope; const bool coarse = project_coarse_ && !projection_coarse.empty(); launch::parallel_for( "solprop_native.transform.projection.custom", @@ -13719,23 +13834,26 @@ void GeneralWorkspace::project(const std::size_t first_node, } void GeneralWorkspace::form_absorber() { - const bool spatial = !problem_.spatial_absorber_matrix.empty(); - const auto modes = problem_.n_modes; - const auto polarisations = problem_.n_pol_state; + const bool spatial = !problem_.spatial.absorber_matrix.empty(); + const auto modes = problem_.dims.n_modes; + const auto polarisations = problem_.dims.n_pol_state; // Flatten (polarisation,time) together: the same scalar radial matrix acts // independently on each component. The views alias existing FFT work buffers. - if (spatial && problem_.family == FieldFamily::envelope) { + if (spatial && problem_.dims.family == FieldFamily::envelope) { using Matrix = Kokkos::View>; - const Matrix input(modal_complex_.data(), modes, polarisations * problem_.n_to); - const Matrix output(absorber_complex_.data(), modes, polarisations * problem_.n_to); + const Matrix input(modal_complex_.data(), modes, + polarisations * problem_.dims.n_to); + const Matrix output(absorber_complex_.data(), modes, + polarisations * problem_.dims.n_to); selected_gemm(absorber_execution_space_, spatial_absorber_complex_, input, output, Complex{}, true); - } else if (spatial && problem_.family == FieldFamily::field_real) { + } else if (spatial && problem_.dims.family == FieldFamily::field_real) { using Matrix = Kokkos::View>; - const Matrix input(modal_real_.data(), modes, polarisations * problem_.n_to); - const Matrix output(absorber_real_.data(), modes, polarisations * problem_.n_to); + const Matrix input(modal_real_.data(), modes, polarisations * problem_.dims.n_to); + const Matrix output(absorber_real_.data(), modes, + polarisations * problem_.dims.n_to); selected_gemm(absorber_execution_space_, spatial_absorber_real_, input, output, 0.0, true); } @@ -13745,19 +13863,20 @@ void GeneralWorkspace::form_absorber() { const auto modal_complex = modal_complex_; const auto absorber_real = absorber_real_; const auto absorber_complex = absorber_complex_; - const auto family = problem_.family; + const auto family = problem_.dims.family; launch::parallel_for( "solprop_native.absorber.nonlinear_term", - Range2(temporal_execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_to}), + Range2(temporal_execution_space_, {0, 0}, + {problem_.n_columns(), problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - if (family == FieldFamily::envelope) + if (family == FieldFamily::envelope) { absorber_complex(column, time) = (spatial ? absorber_complex(column, time) : Complex{}) - 0.5 * rate(time) * modal_complex(column, time); - else if (family == FieldFamily::field_real) + } else if (family == FieldFamily::field_real) { absorber_real(column, time) = (spatial ? absorber_real(column, time) : 0.0) - 0.5 * rate(time) * modal_real(column, time); - else { + } else { double transverse = 0.0; if (spatial) for (std::size_t source = 0; source < modes; ++source) @@ -13773,12 +13892,12 @@ void GeneralWorkspace::form_absorber() { void GeneralWorkspace::finish_stage(const std::size_t stage) { const auto terms = stage_terms(stage); - const double solution_weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; + const double solution_weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // The host's blocks of bins, as the stage's entry kernel takes them. if (terms.blocks_vectorise()) { - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const std::size_t blocks = (n + stage_block - 1) / stage_block; launch::parallel_for( "solprop_native.stage.finish_accumulate", @@ -13794,7 +13913,7 @@ void GeneralWorkspace::finish_stage(const std::size_t stage) { #endif launch::parallel_for( "solprop_native.stage.finish_accumulate", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { terms.finish(stage, solution_weight, error_weight, column, index); }); @@ -13847,15 +13966,15 @@ GeneralWorkspace::form_flow_actions(const std::size_t section, const double dz, std::vector GeneralWorkspace::weighted_error(const std::vector& scale) { require_open(); - if (!problem_.weighted_controller) + if (!problem_.stepper.weighted_controller) throw std::logic_error("this state has no weighted controller workspace"); - require_extent(scale.size(), problem_.n_omega, "weighted controller scale"); + require_extent(scale.size(), problem_.dims.n_omega, "weighted controller scale"); copy_vector(execution_space_, weighted_scale_, scale); const auto residual = residual_; const auto in_norm = norm_mask_; const auto native_scale = weighted_scale_; const auto partials = weighted_partials_; - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; using TeamPolicy = Kokkos::TeamPolicy; using TeamMember = TeamPolicy::member_type; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) @@ -13909,9 +14028,9 @@ void GeneralWorkspace::fill_statistics_time(const bool windowed, const double lo const auto spectrum = statistics_spectrum_; const auto omega = omega_; const auto power_weight = statistics_power_weight_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; launch::parallel_for( "solprop_native.statistics.prepare_analytic", Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), @@ -13943,10 +14062,10 @@ void GeneralWorkspace::form_statistics_power() { const auto weighted_field = statistics_weighted_time_; const bool unit_weight = statistics_unit_weight_; const auto power = statistics_power_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; launch::parallel_for( "solprop_native.statistics.power", - Range2(execution_space_, {0, 0}, {problem_.n_modes, problem_.n_t}), + Range2(execution_space_, {0, 0}, {problem_.dims.n_modes, problem_.dims.n_t}), KOKKOS_LAMBDA(const std::size_t mode, const std::size_t time) { double sum = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) @@ -13961,10 +14080,10 @@ void GeneralWorkspace::form_statistics_power() { void GeneralWorkspace::form_statistics_total_power() { const auto power = statistics_power_; const auto total = statistics_total_power_; - const auto modes = problem_.n_modes; + const auto modes = problem_.dims.n_modes; launch::parallel_for( "solprop_native.statistics.total_power", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), KOKKOS_LAMBDA(const std::size_t time) { double sum = 0.0; for (std::size_t mode = 0; mode < modes; ++mode) @@ -13978,19 +14097,19 @@ void GeneralWorkspace::scan_plasma_statistics(const std::size_t first_node, const RealView2& output, const bool diagnose, const double preionisation) { - if (problem_.envelope_plasma) { + if (problem_.plasma.envelope) { if (diagnose) throw std::invalid_argument("the plasma integration diagnostics read the " "field-grid plasma column, and this problem " "carries none"); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) envelope_statistics(channel, first_node, node_count, output, node_complex_, problem_.effective_node_pol()); return; } // solprop SEP 18: the band rule's statistics, and every problem's run diagnostics, // come from the band scan; the trapezoid's own scan below is unchanged. - if (problem_.plasma_band || diagnose) { + if (problem_.plasma.band || diagnose) { if (diagnose) team_band_statistics(first_node, node_count, output, preionisation); else @@ -13999,28 +14118,30 @@ void GeneralWorkspace::scan_plasma_statistics(const std::size_t first_node, } const auto node_real = node_real_; const auto node_complex = node_complex_; - const bool real_family = problem_.family == FieldFamily::field_real; + const bool real_family = problem_.dims.family == FieldFamily::field_real; const auto coefficients = rate_coefficients_; const auto n_pol = problem_.effective_node_pol(); - const auto n_to = problem_.n_to; - const auto intervals = problem_.rate_intervals; - const auto dt = problem_.dt_fine; - const auto log_field_min = problem_.log_field_min; - const auto d_log_field = problem_.d_log_field; - const auto field_min = problem_.field_min; - const auto field_max = problem_.field_max; - const auto rate_max = problem_.rate_max; - if (problem_.n_channels > 1) { + const auto n_to = problem_.dims.n_to; + const auto intervals = problem_.plasma.rate_intervals; + const auto dt = problem_.grid.dt_fine; + const auto log_field_min = problem_.plasma.log_field_min; + const auto d_log_field = problem_.plasma.d_log_field; + const auto field_min = problem_.plasma.field_min; + const auto field_max = problem_.plasma.field_max; + const auto rate_max = problem_.plasma.rate_max; + if (problem_.plasma.n_channels > 1) { const auto parameters = channel_parameters_; const auto channel_coefficients = channel_coefficients_; - const auto channels = problem_.n_channels; + const auto channels = problem_.plasma.n_channels; launch::parallel_for( "solprop_native.plasma.channel_statistics_scan", Kokkos::RangePolicy(response_space_for(node_count), 0, node_count), KOKKOS_LAMBDA(const std::size_t node) { for (std::size_t channel = 0; channel < channels; ++channel) { - double exposure = 0.0, previous_rate = 0.0, largest_rate = 0.0; + double exposure = 0.0; + double previous_rate = 0.0; + double largest_rate = 0.0; for (std::size_t time = 0; time < n_to; ++time) { const double first = (real_family ? node_real(node * n_pol, time) @@ -14144,16 +14265,16 @@ std::vector GeneralWorkspace::statistics( const std::vector& power_weight) { require_open(); const auto& selected_weight = - power_weight.empty() ? problem_.power_weight : power_weight; - statistics_unit_weight_ = std::all_of(selected_weight.begin(), selected_weight.end(), - [](double value) { return value == 1.0; }); + power_weight.empty() ? problem_.grid.power_weight : power_weight; + statistics_unit_weight_ = + std::ranges::all_of(selected_weight, [](double value) { return value == 1.0; }); if (!selected_weight.empty()) { require_extent(selected_weight.size(), statistics_power_weight_.size(), "statistics power weight"); auto host_weight = Kokkos::create_mirror_view(statistics_power_weight_); for (std::size_t column = 0; column < problem_.n_columns(); ++column) - for (std::size_t index = 0; index < problem_.n_omega; ++index) { - const double value = selected_weight[column * problem_.n_omega + index]; + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) { + const double value = selected_weight[column * problem_.dims.n_omega + index]; if (!(value > 0.0) || !std::isfinite(value)) throw std::invalid_argument( "statistics power weight must be positive and finite"); @@ -14204,35 +14325,35 @@ std::vector GeneralWorkspace::statistics( const auto energy_window_offset = reserve(statistics_energy_window, checked_product(n_windows, problem_.n_columns(), "windowed-energy output")); - const auto peak_offset = reserve(statistics_peak_power, problem_.n_modes); - const auto peak_window_offset = - reserve(statistics_peak_power_window, - checked_product(n_windows, problem_.n_modes, "windowed-peak output")); + const auto peak_offset = reserve(statistics_peak_power, problem_.dims.n_modes); + const auto peak_window_offset = reserve( + statistics_peak_power_window, + checked_product(n_windows, problem_.dims.n_modes, "windowed-peak output")); const auto total_peak_offset = reserve(statistics_peak_power_total, 1); const auto density_offset = reserve(statistics_electron_density, 1); const auto rate_offset = reserve(statistics_peak_ionisation_rate, 1); const auto intensity_offset = reserve(statistics_peak_intensity, 1); const auto quadrature_offset = reserve(statistics_quadrature_error, 1); - const auto fwhm_offset = reserve(statistics_fwhm, problem_.n_t); - const auto populations_offset = - reserve(statistics_populations, - checked_product(2, - checked_product(problem_.n_channels, problem_.n_nodes, - "population output"), - "population output")); + const auto fwhm_offset = reserve(statistics_fwhm, problem_.dims.n_t); + const auto populations_offset = reserve( + statistics_populations, + checked_product(2, + checked_product(problem_.plasma.n_channels, problem_.dims.n_nodes, + "population output"), + "population output")); // solprop SEP 18's run diagnostics, packed last: six maxima over every node. const auto diagnostics_offset = reserve(statistics_plasma_integration, plasma_diagnostic_parts); const bool diagnose = diagnostics_offset != absent; - if (populations_offset != absent || (diagnose && problem_.n_channels > 1)) { - require_extent(channel_preionisation.size(), problem_.n_channels, + if (populations_offset != absent || (diagnose && problem_.plasma.n_channels > 1)) { + require_extent(channel_preionisation.size(), problem_.plasma.n_channels, "statistics channel preionisation"); for (double value : channel_preionisation) if (!std::isfinite(value) || value < 0.0 || value > 1.0) throw std::invalid_argument( "statistics channel preionisation must be in [0, 1]"); auto initial = Kokkos::create_mirror_view(stage_channel_preionisation_); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) initial(0, channel) = channel_preionisation[channel]; graph::deep_copy(execution_space_, stage_channel_preionisation_, initial); } @@ -14247,14 +14368,14 @@ std::vector GeneralWorkspace::statistics( Kokkos::Experimental::quiet_NaN_v); if (omega0_offset != absent || energy_offset != absent) { - if (omega_.empty() || !(problem_.dt > 0.0)) + if (omega_.empty() || !(problem_.grid.dt > 0.0)) throw std::logic_error("native statistics need the coarse frequency axis and dt"); const auto current = current_; const auto omega = omega_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; + const bool envelope = problem_.dims.family == FieldFamily::envelope; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) const StatisticsTeamPolicy policy(execution_space_, static_cast(problem_.n_columns()), @@ -14329,15 +14450,15 @@ std::vector GeneralWorkspace::statistics( } if (energy_window_offset != absent) { - if (omega_.empty() || !(problem_.dt > 0.0)) + if (omega_.empty() || !(problem_.grid.dt > 0.0)) throw std::logic_error("native statistics need the coarse frequency axis and dt"); const auto current = current_; const auto omega = omega_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; const auto columns = problem_.n_columns(); - const bool envelope = problem_.family == FieldFamily::envelope; + const bool envelope = problem_.dims.family == FieldFamily::envelope; for (std::size_t window = 0; window < n_windows; ++window) { const double lower = windows[2 * window]; const double upper = windows[2 * window + 1]; @@ -14395,10 +14516,10 @@ std::vector GeneralWorkspace::statistics( } if (peak_offset != absent) { const auto power = statistics_power_; - const auto n_t = problem_.n_t; + const auto n_t = problem_.dims.n_t; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) const StatisticsTeamPolicy policy(execution_space_, - static_cast(problem_.n_modes), + static_cast(problem_.dims.n_modes), SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); launch::parallel_for( "solprop_native.statistics.device_pack_peak_power", policy, @@ -14419,7 +14540,7 @@ std::vector GeneralWorkspace::statistics( #else launch::parallel_for( "solprop_native.statistics.pack_peak_power", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_modes), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_modes), KOKKOS_LAMBDA(const std::size_t mode) { double peak = 0.0; for (std::size_t time = 0; time < n_t; ++time) @@ -14433,7 +14554,7 @@ std::vector GeneralWorkspace::statistics( form_statistics_total_power(); if (total_peak_offset != absent) { const auto total = statistics_total_power_; - const auto n_t = problem_.n_t; + const auto n_t = problem_.dims.n_t; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) const StatisticsTeamPolicy policy(execution_space_, 1, SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); @@ -14467,13 +14588,13 @@ std::vector GeneralWorkspace::statistics( } if (fwhm_offset != absent) { const auto destination = Kokkos::subview( - packed, std::make_pair(fwhm_offset, fwhm_offset + problem_.n_t)); + packed, std::make_pair(fwhm_offset, fwhm_offset + problem_.dims.n_t)); graph::deep_copy(execution_space_, destination, statistics_total_power_); } } if (intensity_offset != absent) { - if (on_axis.empty() || on_axis.size() % problem_.n_modes != 0 || - on_axis.size() / problem_.n_modes > 2) + if (on_axis.empty() || on_axis.size() % problem_.dims.n_modes != 0 || + on_axis.size() / problem_.dims.n_modes > 2) throw std::invalid_argument( "on-axis synthesis must have one or two rows and n_modes columns"); graph::deep_copy(execution_space_, statistics_on_axis_, 0.0); @@ -14484,10 +14605,10 @@ std::vector GeneralWorkspace::statistics( const auto field = statistics_time_; const auto weighted_field = statistics_weighted_time_; const bool unit_weight = statistics_unit_weight_; - const auto rows = on_axis.size() / problem_.n_modes; - const auto modes = problem_.n_modes; - const auto n_pol = problem_.n_pol_state; - const auto n_t = problem_.n_t; + const auto rows = on_axis.size() / problem_.dims.n_modes; + const auto modes = problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_t = problem_.dims.n_t; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) const StatisticsTeamPolicy policy(execution_space_, 1, SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); @@ -14532,7 +14653,8 @@ std::vector GeneralWorkspace::statistics( double power = 0.0; for (std::size_t row = 0; row < rows; ++row) { for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex value{}, weighted_value{}; + Complex value{}; + Complex weighted_value{}; for (std::size_t mode = 0; mode < modes; ++mode) { value += matrix(row * modes + mode) * field(mode * n_pol + pol, time); if (!unit_weight) @@ -14556,8 +14678,8 @@ std::vector GeneralWorkspace::statistics( fill_statistics_pair(true, windows[2 * window], windows[2 * window + 1]); form_statistics_power(); const auto power = statistics_power_; - const auto n_t = problem_.n_t; - const auto modes = problem_.n_modes; + const auto n_t = problem_.dims.n_t; + const auto modes = problem_.dims.n_modes; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) const StatisticsTeamPolicy policy(execution_space_, static_cast(modes), SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); @@ -14595,9 +14717,9 @@ std::vector GeneralWorkspace::statistics( if (density_offset != absent || rate_offset != absent || populations_offset != absent || diagnose) { - if (!problem_.plasma) + if (!problem_.plasma.enabled) throw std::logic_error("this problem declares no compiled plasma response"); - if (diagnose && problem_.envelope_plasma) + if (diagnose && problem_.plasma.envelope) throw std::invalid_argument("the plasma integration diagnostics read the " "field-grid plasma column, and this problem " "carries none"); @@ -14620,22 +14742,23 @@ std::vector GeneralWorkspace::statistics( } else { statistics_modal_time(); graph::deep_copy(execution_space_, stage_taper_, taper); - scale_transform(problem_.n_stages - 1, taper); - for (std::size_t first = 0; first < problem_.n_nodes; - first += problem_.node_chunk) { - const auto count = std::min(problem_.node_chunk, problem_.n_nodes - first); + scale_transform(problem_.stepper.n_stages - 1, taper); + for (std::size_t first = 0; first < problem_.dims.n_nodes; + first += problem_.dims.node_chunk) { + const auto count = + std::min(problem_.dims.node_chunk, problem_.dims.n_nodes - first); synthesise(first, count); scan_plasma_statistics(first, count, source, diagnose, preionisation); } } } if (populations_offset != absent) { - const bool envelope = problem_.envelope_plasma; - const auto channels = problem_.n_channels; + const bool envelope = problem_.plasma.envelope; + const auto channels = problem_.plasma.n_channels; const auto initial = stage_channel_preionisation_; launch::parallel_for( "solprop_native.statistics.pack_populations", - Range2(execution_space_, {0, 0}, {problem_.n_nodes, channels}), + Range2(execution_space_, {0, 0}, {problem_.dims.n_nodes, channels}), KOKKOS_LAMBDA(const std::size_t node, const std::size_t channel) { const auto offset = populations_offset + 2 * (node * channels + channel); packed(offset) = @@ -14646,7 +14769,7 @@ std::vector GeneralWorkspace::statistics( packed(offset + 1) = source(node, 2 * channel + 1); }); } - const auto nodes = problem_.n_nodes; + const auto nodes = problem_.dims.n_nodes; launch::parallel_for( "solprop_native.statistics.pack_plasma", Kokkos::RangePolicy(execution_space_, 0, 1), @@ -14716,9 +14839,9 @@ std::vector GeneralWorkspace::whitened_statistics( throw std::invalid_argument("a general modal basis's statistics are omega0, the " "energy, the peak total power and the widths"); const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto modes = problem_.n_modes; - const auto n_pol = problem_.n_pol_state; + const auto n = problem_.dims.n_omega; + const auto modes = problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; const bool whiten = !root.empty(); if (whiten) { require_extent( @@ -14819,8 +14942,8 @@ void GeneralWorkspace::require_flux_closure() { return; execution_space_.fence("solprop_native flux closure allocation"); if (time) - statistics_weighted_time_ = - ComplexView2("solprop_native.statistics.weighted_time", rows, problem_.n_t); + statistics_weighted_time_ = ComplexView2("solprop_native.statistics.weighted_time", + rows, problem_.dims.n_t); if (magnitude) statistics_magnitude_ = RealView2("solprop_native.statistics.flux_magnitude", @@ -14841,10 +14964,10 @@ void GeneralWorkspace::fill_statistics_closure() { const auto omega = omega_; const auto active = active_mask_; const auto flux = flux_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto n_pol = problem_.n_pol_state; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto n_pol = problem_.dims.n_pol_state; + const bool envelope = problem_.dims.family == FieldFamily::envelope; launch::parallel_for( "solprop_native.statistics.prepare_closure", Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), @@ -14881,13 +15004,13 @@ void GeneralWorkspace::separable_time_statistics(const std::size_t axis_peak, const auto flux = flux_; const auto active = active_mask_; const auto omega = omega_; - const auto n_modes = problem_.n_modes; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; + const auto n_modes = problem_.dims.n_modes; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; const auto layout = transverse_layout(); - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; if (axis_peak != absent) { // The axis's closure first, kept in the closure rows, then its field: each the // coherent on-axis sum of the analytic spectrum (solprop SEP 34). On free space the @@ -14945,7 +15068,7 @@ void GeneralWorkspace::separable_time_statistics(const std::size_t axis_peak, if (peak == absent && !fluence) return; const auto transverse_count = - static_cast(problem_.slab_geometry() ? problem_.n_y : n_modes); + static_cast(problem_.slab_geometry() ? problem_.spatial.n_y : n_modes); launch::parallel_for( "solprop_native.free.statistics_transverse_input", Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), @@ -14964,12 +15087,12 @@ void GeneralWorkspace::separable_time_statistics(const std::size_t axis_peak, if (peak == absent) return; const bool slab = problem_.slab_geometry(); - const auto n_x = problem_.n_x; - const auto n_p = problem_.n_p; + const auto n_x = problem_.spatial.n_x; + const auto n_p = problem_.spatial.n_p; // The planar synthesis carries the temporal pad scale; the statistics read the // coarse field, so it is divided out (solprop's `collect_spatial`). const double crop_scale = - static_cast(problem_.n_t) / static_cast(problem_.n_to); + static_cast(problem_.dims.n_t) / static_cast(problem_.dims.n_to); for (std::size_t unit = 0; unit < transverse_units(); unit += transverse_units_per_chunk()) { const auto chunk = transverse_chunk(unit); @@ -15057,19 +15180,19 @@ void GeneralWorkspace::separable_fluence(const double intensity_factor) { const auto scratch = transverse_; const auto closure_scratch = flux_transverse_; const auto partials = spatial_statistics_partials_; - const PhysicalSpectrum spectrum{transverse_layout(), - synthesis_, - problem_.n_x, - problem_.n_p, - static_cast(problem_.n_t) / - static_cast(problem_.n_to), - problem_.slab_geometry()}; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const bool envelope = problem_.family == FieldFamily::envelope; + const PhysicalSpectrum spectrum{.layout = transverse_layout(), + .synthesis = synthesis_, + .n_x = problem_.spatial.n_x, + .n_p = problem_.spatial.n_p, + .crop_scale = static_cast(problem_.dims.n_t) / + static_cast(problem_.dims.n_to), + .slab = problem_.slab_geometry()}; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const bool envelope = problem_.dims.family == FieldFamily::envelope; const double measure = - intensity_factor * problem_.dt / static_cast(problem_.n_t); + intensity_factor * problem_.grid.dt / static_cast(problem_.dims.n_t); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // One node per thread, whole planes per block: adjacent threads read adjacent cells. launch::parallel_for( @@ -15104,7 +15227,9 @@ void GeneralWorkspace::separable_fluence(const double intensity_factor) { KOKKOS_LAMBDA(const std::size_t item) { const std::size_t first = item * run; const std::size_t count = n_nodes - first < run ? n_nodes - first : run; - double sum[run] = {}, electric_size[run] = {}, magnetic_size[run] = {}; + double sum[run] = {}; + double electric_size[run] = {}; + double magnetic_size[run] = {}; for (std::size_t index = 0; index < n; ++index) { const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { @@ -15138,8 +15263,8 @@ void GeneralWorkspace::separable_fluence(const double intensity_factor) { // work. void GeneralWorkspace::plane_fluence_moments(double (&moments)[plane_moment_slots]) { const auto partials = spatial_statistics_partials_; - const auto n_nodes = problem_.n_nodes; - const auto n_x = problem_.n_x; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_x = problem_.spatial.n_x; const Kokkos::RangePolicy nodes(execution_space_, 0, n_nodes); std::size_t nonfinite = 0; launch::parallel_reduce( @@ -15187,8 +15312,9 @@ void GeneralWorkspace::plane_fluence_moments(double (&moments)[plane_moment_slot const double sample = Kokkos::max(partials(node, 0), 0.0) / maximum; partials(node, 0) = sample; update.total += sample; + const std::size_t row = node / n_x; update.x += sample * static_cast(node % n_x); - update.y += sample * static_cast(node / n_x); + update.y += sample * static_cast(row); }, Kokkos::Sum(sums)); moments[0] = worst.val; @@ -15229,8 +15355,8 @@ std::vector GeneralWorkspace::plane_fluence_trace( } const auto packed = statistics_packed_; const auto samples = spatial_statistics_partials_; - const auto n_x = problem_.n_x; - const auto n_y = problem_.n_y; + const auto n_x = problem_.spatial.n_x; + const auto n_y = problem_.spatial.n_y; const auto first_ring = n_radii + 1; if (n_angles != 0) { std::vector rings(cosines); @@ -15242,15 +15368,16 @@ std::vector GeneralWorkspace::plane_fluence_trace( double second = 0.0; launch::parallel_reduce( "solprop_native.free.plane_fluence_second", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_nodes), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_nodes), KOKKOS_LAMBDA(const std::size_t node, double& update) { const double x = (static_cast(node % n_x) * dx - center_x) / extent; - const double y = (static_cast(node / n_x) * dy - center_y) / extent; + const std::size_t row = node / n_x; + const double y = (static_cast(row) * dy - center_y) / extent; update += samples(node, 0) * (x * x + y * y); }, Kokkos::Sum(second)); - const double last_x = static_cast(n_x - 1); - const double last_y = static_cast(n_y - 1); + const auto last_x = static_cast(n_x - 1); + const auto last_y = static_cast(n_y - 1); // One thread per ring: a ring's samples are few, and a thread's sum needs no // reduction. launch::parallel_for( @@ -15305,14 +15432,15 @@ GeneralWorkspace::axis_spectrum(const std::vector& matrix) { if (!initialized_) throw std::logic_error("no initial field has been loaded"); if (problem_.separable_geometry() || matrix.empty() || - matrix.size() % problem_.n_modes != 0 || - matrix.size() / problem_.n_modes > statistics_on_axis_.size() / problem_.n_modes) + matrix.size() % problem_.dims.n_modes != 0 || + matrix.size() / problem_.dims.n_modes > + statistics_on_axis_.size() / problem_.dims.n_modes) throw std::invalid_argument( "an on-axis row needs a modal state and at most two rows of n_modes columns"); - const auto n_rows = matrix.size() / problem_.n_modes; - const auto n_pol = problem_.n_pol_state; - const auto n_modes = problem_.n_modes; - const auto n = problem_.n_omega; + const auto n_rows = matrix.size() / problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_modes = problem_.dims.n_modes; + const auto n = problem_.dims.n_omega; const auto columns = checked_product(n_rows, n_pol, "on-axis columns"); const auto samples = checked_product(columns, n, "on-axis samples"); const auto packed_size = checked_product(2, samples, "on-axis samples"); @@ -15359,19 +15487,19 @@ void GeneralWorkspace::statistics_modal_time() { const auto current = current_; const auto fine = fine_spectrum_; const auto crop = crop_index_; - const auto inverse_n_to = 1.0 / static_cast(problem_.n_to); - const auto n_to = problem_.n_to; - const bool analytic = problem_.family == FieldFamily::field_analytic; + const auto inverse_n_to = 1.0 / static_cast(problem_.dims.n_to); + const auto n_to = problem_.dims.n_to; + const bool analytic = problem_.dims.family == FieldFamily::field_analytic; launch::parallel_for( "solprop_native.plasma.statistics_pad", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { const double analytic_scale = analytic && index > 0 && 2 * crop(index) < n_to ? 2.0 : 1.0; fine(column, crop(index)) = analytic_scale * inverse_n_to * current(column, index); }); - if (problem_.family == FieldFamily::field_real) + if (problem_.dims.family == FieldFamily::field_real) graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, KokkosFFT::Normalization::none); else if (!retained_synthesis_) @@ -15396,15 +15524,15 @@ GeneralWorkspace::axis_populations(const std::vector& matrix, require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - if (!problem_.plasma || problem_.n_channels == 0) + if (!problem_.plasma.enabled || problem_.plasma.n_channels == 0) throw std::invalid_argument( "the on-axis plasma statistics need a compiled plasma response"); - if (problem_.separable_geometry() || problem_.general_modal || retained_synthesis_) + if (problem_.separable_geometry() || problem_.general.enabled || retained_synthesis_) throw std::invalid_argument( "the on-axis plasma statistics read a resolved modal state"); ++axis_scans_; - const auto n_modes = problem_.n_modes; - const auto n_pol_state = problem_.n_pol_state; + const auto n_modes = problem_.dims.n_modes; + const auto n_pol_state = problem_.dims.n_pol_state; const auto node_pol = problem_.effective_node_pol(); if (matrix.empty() || matrix.size() % n_modes != 0 || matrix.size() / n_modes > 2) throw std::invalid_argument( @@ -15423,20 +15551,20 @@ GeneralWorkspace::axis_populations(const std::vector& matrix, "an on-axis row has one component on each of the node scan's"); // A pair on a field grid is combined into its magnitude in the node buffer; an // envelope's is kept, x and y, in its own row. - const bool ellipse = pair && problem_.envelope_plasma; + const bool ellipse = pair && problem_.plasma.envelope; const bool combine = pair && !ellipse; if (ellipse && statistics_axis_row_.extent(0) == 0) { execution_space_.fence("solprop_native on-axis row allocation"); statistics_axis_row_ = - ComplexView2("solprop_native.statistics.axis_row", 2, problem_.n_to); + ComplexView2("solprop_native.statistics.axis_row", 2, problem_.dims.n_to); } - require_extent(channel_preionisation.size(), problem_.n_channels, + require_extent(channel_preionisation.size(), problem_.plasma.n_channels, "on-axis channel preionisation"); for (const double value : channel_preionisation) if (!std::isfinite(value) || value < 0.0 || value > 1.0) throw std::invalid_argument("on-axis channel preionisation must be in [0, 1]"); auto initial_host = Kokkos::create_mirror_view(stage_channel_preionisation_); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) initial_host(0, channel) = channel_preionisation[channel]; graph::deep_copy(execution_space_, stage_channel_preionisation_, initial_host); const std::vector held( @@ -15452,11 +15580,11 @@ GeneralWorkspace::axis_populations(const std::vector& matrix, const auto node_real = node_real_; const auto node_complex = node_complex_; const auto axis_row = statistics_axis_row_; - const bool real_family = problem_.family == FieldFamily::field_real; + const bool real_family = problem_.dims.family == FieldFamily::field_real; const auto written = combine ? std::size_t{1} : components; launch::parallel_for( "solprop_native.statistics.axis_row", - Range2(execution_space_, {0, 0}, {written, problem_.n_to}), + Range2(execution_space_, {0, 0}, {written, problem_.dims.n_to}), KOKKOS_LAMBDA(const std::size_t component, const std::size_t time) { // Component `row * n_pol_state + pol`, `passes.synthesise`'s layout. const auto sum_real = [&](const std::size_t row, const std::size_t pol) { @@ -15492,18 +15620,18 @@ GeneralWorkspace::axis_populations(const std::vector& matrix, }); const auto source = plasma_statistics_scratch_; if (ellipse) - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) envelope_statistics(channel, 0, 1, source, statistics_axis_row_, 2); else scan_plasma_statistics(0, 1, source, false, channel_preionisation[0]); - const auto channels = problem_.n_channels; + const auto channels = problem_.plasma.n_channels; const auto packed_size = 2 * channels; if (statistics_packed_.size() < packed_size) { execution_space_.fence("solprop_native statistics output resize"); statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); } const auto packed = statistics_packed_; - const bool envelope = problem_.envelope_plasma; + const bool envelope = problem_.plasma.envelope; const auto initial = stage_channel_preionisation_; // `statistics`' own packing of a node's populations. launch::parallel_for( @@ -15538,18 +15666,18 @@ std::vector GeneralWorkspace::aperture_statistics( } FarFieldPass::Inputs GeneralWorkspace::far_field_inputs() const { - return FarFieldPass::Inputs{spatial_spectral_weights_, - spatial_on_axis_, - spatial_.square, - spatial_.square_low, - spatial_.transverse, - spatial_.transverse_low, - spatial_.spectral_rate, - spatial_.user_rate, - spatial_.transverse_rate, - omega_, - active_mask_, - spatial_.evanescent}; + return FarFieldPass::Inputs{.weights = spatial_spectral_weights_, + .on_axis = spatial_on_axis_, + .square = spatial_.square, + .square_low = spatial_.square_low, + .transverse = spatial_.transverse, + .transverse_low = spatial_.transverse_low, + .spectral_rate = spatial_.spectral_rate, + .user_rate = spatial_.user_rate, + .transverse_rate = spatial_.transverse_rate, + .omega = omega_, + .active = active_mask_, + .evanescent = spatial_.evanescent}; } std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t flags, @@ -15565,8 +15693,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla intensity_factor <= 0.0 || !std::isfinite(z)) throw std::invalid_argument( "invalid spatial statistic flags, intensity factor or position"); - if ((flags & 32U) != 0 && - (!problem_.free_geometry() || problem_.slab_geometry() || problem_.n_y < 2)) + if ((flags & 32U) != 0 && (!problem_.free_geometry() || problem_.slab_geometry() || + problem_.spatial.n_y < 2)) throw std::invalid_argument("a plane's fluence moments need a Cartesian plane"); constexpr auto absent = std::numeric_limits::max(); constexpr double lowest = std::numeric_limits::lowest(); @@ -15582,12 +15710,12 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla const auto energy_offset = reserve(1, 1); const auto axis_peak_offset = reserve(2, 1); const auto peak_offset = reserve(4, 1); - const auto spectrum_offset = reserve(8, problem_.n_omega); + const auto spectrum_offset = reserve(8, problem_.dims.n_omega); // Widths read the signed fluence at the nodes, the separately reconstructed axis, // which only a radial width reads, and the largest magnitude, which fixes the // fluence's rounding budget (solprop SEP 34). const auto fluence_offset = - reserve(16, checked_sum(problem_.n_nodes, 2, "spatial statistics output")); + reserve(16, checked_sum(problem_.dims.n_nodes, 2, "spatial statistics output")); // On a Cartesian plane the widths reduce here instead, and the moments cross. const auto moments_offset = reserve(32, plane_moment_slots); if (packed_size == 0) @@ -15609,13 +15737,13 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla const auto flux = flux_; const auto active = active_mask_; const auto omega = omega_; - const auto n_modes = problem_.n_modes; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n_modes = problem_.dims.n_modes; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; + const bool envelope = problem_.dims.family == FieldFamily::envelope; if (energy_offset != absent) { launch::parallel_for( "solprop_native.spatial.energy_partials", @@ -15653,7 +15781,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla const bool live = active(index) != 0U; double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}, magnetic{}; + Complex electric{}; + Complex magnetic{}; for (std::size_t mode = 0; mode < n_modes; ++mode) { const Complex value = axis_weights(mode) * current(mode * n_pol + pol, index); @@ -15706,7 +15835,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla KOKKOS_LAMBDA(const std::size_t time) { double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}, magnetic{}; + Complex electric{}; + Complex magnetic{}; for (std::size_t mode = 0; mode < n_modes; ++mode) { electric += axis_weights(mode) * field(mode * n_pol + pol, time); magnetic += axis_weights(mode) * closure(mode * n_pol + pol, time); @@ -15735,8 +15865,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla const auto power = statistics_power_; const auto magnitude = statistics_magnitude_; const auto synthesis = synthesis_; - const double crop_scale = - static_cast(problem_.n_t) / static_cast(problem_.n_to); + const double crop_scale = static_cast(problem_.dims.n_t) / + static_cast(problem_.dims.n_to); // Each node's field and closure, `S F` and `S G` over the modes, as GEMMs over // interleaved real columns (each sample's real and imaginary parts; a mode's // polarisations are adjacent rows): the field's into the spectrum buffer, free @@ -15766,7 +15896,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla "solprop_native.spatial.physical_intensity", Range2(execution_space_, {0, 0}, {n_nodes, n_t}), KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - double total = 0.0, size = 0.0; + double total = 0.0; + double size = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { const std::size_t at = 2 * (pol * n_t + time); const Complex electric = @@ -15786,7 +15917,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla Kokkos::RangePolicy(execution_space_, 0, n_nodes), KOKKOS_LAMBDA(const std::size_t node) { double peak = lowest; - double sum = 0.0, size = 0.0; + double sum = 0.0; + double size = 0.0; for (std::size_t time = 0; time < n_t; ++time) { peak = Kokkos::max(peak, power(node, time)); sum += power(node, time); @@ -15838,11 +15970,11 @@ void GeneralWorkspace::finish(const double dz) { require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - if (problem_.low_storage) { + if (problem_.stepper.low_storage) { // The march accepted the step in place (solprop SEP 24 section 6.2): the // boundary treatments act on the state itself, and the stash a response pass // left is the last evaluation's, at a predicted field, not this one's. - if (problem_.split_step_absorber) + if (problem_.absorber.split_step) split_step(dz, current_); else mask(current_); @@ -15850,7 +15982,7 @@ void GeneralWorkspace::finish(const double dz) { execution_space_.fence("solprop_native accepted-step boundary"); return; } - if (problem_.split_step_absorber) + if (problem_.absorber.split_step) split_step(dz, advanced_); else mask(advanced_); @@ -15865,7 +15997,7 @@ void GeneralWorkspace::finish_quadrature() { const Kokkos::Profiling::ScopedRegion region("solprop_native.quadrature.finish"); if (!coarse_pending_) return; - if (problem_.family == FieldFamily::envelope) + if (problem_.dims.family == FieldFamily::envelope) graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, projected_coarse_spectrum_, KokkosFFT::Normalization::none); else if (forward_pack_coarse_plan_) @@ -15883,7 +16015,7 @@ void GeneralWorkspace::score_quadrature() { quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; quadrature_stale_ = false; if (quadrature_partials_.empty() || - (!problem_.nonlinear && !problem_.plasma && !dispersive_)) + (!problem_.kerr.nonlinear && !problem_.plasma.enabled && !dispersive_)) return; finish_quadrature(); // SEP 6's dispersive products join the embedded rule's spectrum once, when scored. @@ -15897,9 +16029,9 @@ void GeneralWorkspace::score_quadrature() { const auto prefactor_terms = prefactor_terms_; const auto prefactor_weights = stage_prefactor_; const auto partials = quadrature_partials_; - const auto last = problem_.n_stages - 1; - const auto n = problem_.n_omega; - const auto n_prefactor = problem_.n_prefactor_terms; + const auto last = problem_.stepper.n_stages - 1; + const auto n = problem_.dims.n_omega; + const auto n_prefactor = problem_.linear.n_prefactor_terms; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) using TeamPolicy = Kokkos::TeamPolicy; using TeamMember = TeamPolicy::member_type; @@ -16017,10 +16149,10 @@ void GeneralWorkspace::promote_fsal() { const auto stages = stages_; const auto whole = whole_propagator_; const auto recycled = recycled_; - const auto last = problem_.n_stages - 1; + const auto last = problem_.stepper.n_stages - 1; launch::parallel_for( "solprop_native.stage.promote_fsal", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { recycled(column, index) = whole(column, index) * stages(last, column, index); }); @@ -16041,7 +16173,7 @@ void GeneralWorkspace::mask(const ComplexView2& values) { const auto active = active_mask_; launch::parallel_for( "solprop_native.mask.accepted", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { if (active(index) == 0U) values(column, index) = Complex{}; @@ -16104,7 +16236,7 @@ void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, const bool tail = temporal_tail_rows_ != 0 && active_rows == temporal_tail_rows_; if (active_rows == 0) active_rows = coarse_spectrum_.extent(0); - if (problem_.family == FieldFamily::envelope) { + if (problem_.dims.family == FieldFamily::envelope) { if (tail) coarse_tail_inverse_complex_.execute(); else @@ -16115,7 +16247,7 @@ void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, 1.0); split_apply("solprop_native.absorber.split_apply_complex", execution_space_, coarse_time_complex_, absorber_coarse_, split_factor_, dz, active_rows, - problem_.n_t); + problem_.dims.n_t); if (tail) coarse_tail_forward_complex_.execute(); else @@ -16131,7 +16263,7 @@ void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, add_split_removal(dz, active_rows, parseval, first_column, {}, 1.0); split_apply("solprop_native.absorber.split_apply_real", execution_space_, coarse_time_real_, absorber_coarse_, split_factor_, dz, active_rows, - problem_.n_t); + problem_.dims.n_t); if (tail) coarse_tail_forward_real_.execute(); else @@ -16147,7 +16279,7 @@ void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, // one pass after the blocks. Equal to the reference's split in real space up to the // transforms' rounding, at any block size. void GeneralWorkspace::split_state_rows(const double dz, const ComplexView2& state) { - const double inverse_time = 1.0 / static_cast(problem_.n_t); + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); const auto columns = problem_.n_columns(); for (std::size_t first = 0; first < columns; first += split_rows_) { const auto rows = std::min(split_rows_, columns - first); @@ -16159,7 +16291,8 @@ void GeneralWorkspace::split_state_rows(const double dz, const ComplexView2& sta if (monitoring_split()) add_split_removal(dz, rows, 0.0, first, block, inverse_time * inverse_time); split_apply("solprop_native.absorber.split_apply_complex", execution_space_, block, - absorber_coarse_, split_factor_, dz, rows, problem_.n_t, inverse_time); + absorber_coarse_, split_factor_, dz, rows, problem_.dims.n_t, + inverse_time); graph::execute(tail ? *split_tail_forward_plan_ : *split_forward_plan_, block, block, KokkosFFT::Normalization::none); } @@ -16191,14 +16324,15 @@ void GeneralWorkspace::add_split_removal(const double dz, const std::size_t acti // formed once, not per row. launch::parallel_for( "solprop_native.monitor.split_share", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), KOKKOS_LAMBDA(const std::size_t index) { share(index) = -Kokkos::expm1(-rate(index) * dz); half(index) = Kokkos::exp(-0.5 * rate(index) * dz); }); const auto guards = monitor_weights_; - const auto samples = Range2(execution_space_, {0, 0}, {active_rows, problem_.n_t}); - if (problem_.family == FieldFamily::envelope) { + const auto samples = + Range2(execution_space_, {0, 0}, {active_rows, problem_.dims.n_t}); + if (problem_.dims.family == FieldFamily::envelope) { launch::parallel_reduce( "solprop_native.monitor.split_removal_complex", samples, KOKKOS_LAMBDA(const std::size_t row, const std::size_t index, @@ -16249,11 +16383,12 @@ void GeneralWorkspace::add_split_removal(const double dz, const std::size_t acti void GeneralWorkspace::bind_monitor(const std::vector& weights, const std::vector& column_weights) { require_open(); - const auto n_t = problem_.n_t; + const auto n_t = problem_.dims.n_t; const auto columns = problem_.n_columns(); require_extent(weights.size(), 4 * n_t, "monitor weights"); if (!column_weights.empty()) - require_extent(column_weights.size(), problem_.n_modes, "monitor column weights"); + require_extent(column_weights.size(), problem_.dims.n_modes, + "monitor column weights"); if (monitor_weights_.extent(0) == 0) { monitor_weights_ = RealView2("solprop_native.monitor.weights", 4, n_t); monitor_column_weights_ = @@ -16276,7 +16411,7 @@ void GeneralWorkspace::bind_monitor(const std::vector& weights, std::vector expanded(columns, 1.0); if (!column_weights.empty()) for (std::size_t column = 0; column < columns; ++column) - expanded[column] = column_weights[column / problem_.n_pol_state]; + expanded[column] = column_weights[column / problem_.dims.n_pol_state]; copy_vector(execution_space_, monitor_column_weights_, expanded); graph::deep_copy(execution_space_, monitor_removed_, 0.0); graph::deep_copy(execution_space_, monitor_found_, SplitSums{}); @@ -16324,6 +16459,7 @@ MonitorSums monitor_sums(const char* label, const ExecutionSpace& execution, double total = 0.0; double regions[4] = {0.0, 0.0, 0.0, 0.0}; for (std::size_t index = first; index < last; ++index) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide const double value = power(row, index); total += value; for (std::size_t region = 0; region < 4; ++region) @@ -16464,7 +16600,7 @@ void GeneralWorkspace::bind_edge(const std::vector& faces, require_extent(nodes, transverse_units(), "edge nodes"); require_extent(synthesis.size(), 0, "a separable edge's synthesis"); } else { - require_extent(nodes, problem_.n_modes, "edge nodes"); + require_extent(nodes, problem_.dims.n_modes, "edge nodes"); require_extent(synthesis.size(), nodes * nodes, "edge synthesis"); } if (first_row >= nodes) @@ -16473,14 +16609,14 @@ void GeneralWorkspace::bind_edge(const std::vector& faces, edge_faces_ = RealView2("solprop_native.edge.faces", 4, nodes); edge_clip_ = RealView2("solprop_native.edge.clip", 4, nodes); edge_totals_ = RealView2("solprop_native.edge.totals", 4, nodes); - edge_columns_ = RealView("solprop_native.edge.columns", problem_.n_modes); + edge_columns_ = RealView("solprop_native.edge.columns", problem_.dims.n_modes); if (!synthesis.empty()) edge_synthesis_ = RealView2("solprop_native.edge.synthesis", nodes, nodes); - if (!synthesis.empty() && problem_.n_pol_state == 1 && - current_.extent(1) == problem_.n_omega) + if (!synthesis.empty() && problem_.dims.n_pol_state == 1 && + current_.extent(1) == problem_.dims.n_omega) edge_nodes_ = RealView2( Kokkos::view_alloc(Kokkos::WithoutInitializing, "solprop_native.edge.nodes"), - nodes, 2 * problem_.n_omega); + nodes, 2 * problem_.dims.n_omega); } copy_vector(execution_space_, edge_faces_, faces); copy_vector(execution_space_, edge_clip_, clip); @@ -16507,7 +16643,7 @@ void GeneralWorkspace::bind_edge_lines(const EdgeLines& lines) { if (std::accumulate(lines.counts.begin(), lines.counts.end(), std::size_t{0}) != lines.positions.size()) throw std::invalid_argument("the edge's lines don't account for their nodes"); - const auto planes = problem_.n_pol_state * problem_.n_omega; + const auto planes = problem_.dims.n_pol_state * problem_.dims.n_omega; // A runner's next point reuses the plans of its shapes, and only those stay held. decltype(edge_batches_) batches; std::vector bound; @@ -16517,10 +16653,10 @@ void GeneralWorkspace::bind_edge_lines(const EdgeLines& lines) { EdgeLine line; line.axis = lines.axes[index]; line.nodes = lines.counts[index]; - line.rows = planes * (problem_.slab_geometry() ? problem_.n_p + line.rows = planes * (problem_.slab_geometry() ? problem_.spatial.n_p : !problem_.separable_geometry() ? std::size_t{1} - : line.axis == 1 ? problem_.n_y - : problem_.n_x); + : line.axis == 1 ? problem_.spatial.n_y + : problem_.spatial.n_x); line.first = lines.inward[2 * index]; line.last = lines.inward[2 * index + 1]; line.factor = lines.factors[index]; @@ -16615,8 +16751,8 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, if (edge_faces_.extent(0) == 0) throw std::logic_error("no transverse edge is bound"); const auto current = current_; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; EdgeNumbers reading; const bool separable = problem_.separable_geometry(); if (separable) @@ -16652,7 +16788,7 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, using Interleaved = Kokkos::View>; static_assert(sizeof(Complex) == 2 * sizeof(double)); - const auto nodes = problem_.n_modes; + const auto nodes = problem_.dims.n_modes; const auto first = whole ? std::size_t{0} : edge_first_row_; const auto rows = nodes - first; const Interleaved state(reinterpret_cast(current_.data()), nodes, 2 * n); @@ -16667,8 +16803,8 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, return real * real + imaginary * imaginary; }; const auto sums = [&](const char* label, const RealView2& weights) { - return monitor_sums(label, execution_space_, n, rows, EdgeWeights{weights, first}, - power); + return monitor_sums(label, execution_space_, n, rows, + EdgeWeights{.weights = weights, .first = first}, power); }; take(sums("solprop_native.edge.faces", edge_faces_)); if (whole) @@ -16679,7 +16815,7 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, // Each plane's synthesis on the rows wanted. A launch reads every node: its total // is the grid's. A dense product per sample, on a read's few outer rows. const auto synthesis = edge_synthesis_; - const auto nodes = problem_.n_modes; + const auto nodes = problem_.dims.n_modes; const auto first = whole ? std::size_t{0} : edge_first_row_; const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t row) { const auto pol = plane / n; @@ -16691,7 +16827,7 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, }; const auto sums = [&](const char* label, const RealView2& weights) { return monitor_sums(label, execution_space_, n_pol * n, nodes - first, - EdgeWeights{weights, first}, power); + EdgeWeights{.weights = weights, .first = first}, power); }; take(sums("solprop_native.edge.faces", edge_faces_)); if (whole) @@ -16703,7 +16839,7 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, const auto energies = edge_columns_; launch::parallel_for( "solprop_native.edge.columns", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_modes), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_modes), KOKKOS_LAMBDA(const std::size_t mode) { double sum = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) @@ -16738,16 +16874,30 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, if (strip || want_current) { for (const auto& line : edge_lines_) { if (!separable) { - take_line(line, EdgeRadialValue{edge_synthesis_, current, edge_scale_, n_pol, n, - problem_.n_modes}); + take_line(line, EdgeRadialValue{.synthesis = edge_synthesis_, + .current = current, + .scale = edge_scale_, + .n_pol = n_pol, + .n = n, + .modes = problem_.dims.n_modes}); } else if (problem_.slab_geometry()) { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), 1, 0, 1}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = 1, + .row_stride = 0, + .node_stride = 1}); } else if (line.axis == 1) { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), problem_.n_y, - problem_.n_x, 1}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.spatial.n_y, + .row_stride = problem_.spatial.n_x, + .node_stride = 1}); } else { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), problem_.n_x, - 1, problem_.n_x}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.spatial.n_x, + .row_stride = 1, + .node_stride = problem_.spatial.n_x}); } } } @@ -16765,15 +16915,15 @@ std::vector GeneralWorkspace::monitor_reading() { const auto time = statistics_time_; const auto weights = monitor_weights_; const auto column_weights = monitor_column_weights_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; // A one-sided spectrum's last bin is the Nyquist bin only on an even axis. const bool nyquist = n_t % 2 == 0; const auto columns = problem_.n_columns(); MonitorSums sums; // Half the analytic signal's transform work, on buffers that already exist. - if (problem_.split_step_absorber && !envelope) { + if (problem_.absorber.split_step && !envelope) { const auto coarse = coarse_spectrum_; const auto samples = coarse_time_real_; const double inverse_n = 1.0 / static_cast(n_t); @@ -16797,7 +16947,7 @@ std::vector GeneralWorkspace::monitor_reading() { } } const auto rows = statistics_spectrum_.extent(0); - const bool analytic = !(problem_.split_step_absorber && !envelope); + const bool analytic = !(problem_.absorber.split_step && !envelope); for (std::size_t first = 0; analytic && first < columns; first += rows) { const auto count = std::min(rows, columns - first); // Every bin of the rows read is written, the empty half of a one-sided spectrum @@ -16826,7 +16976,7 @@ std::vector GeneralWorkspace::monitor_reading() { } double removed = 0.0; SplitSums found; - if (problem_.split_step_absorber) { + if (problem_.absorber.split_step) { execution_space_.fence("solprop_native monitor reading"); removed = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, monitor_removed_)(); @@ -16843,7 +16993,7 @@ std::vector GeneralWorkspace::monitor_reading() { // per-thread histograms need no atomics: on a device an atomic per element took // most of a reading, several steps' time on an AMD Instinct MI325X. const auto buckets = occupancy_buckets_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; for (std::size_t first = 0; first < occupancy_count_; first += occupancy_reduction_slots) { OccupancySums occupancy; @@ -16876,7 +17026,7 @@ std::vector GeneralWorkspace::monitor_reading() { void GeneralWorkspace::bind_occupancy(const std::vector& buckets, const std::size_t count) { require_open(); - require_extent(buckets.size(), problem_.n_modes * problem_.n_omega, + require_extent(buckets.size(), problem_.dims.n_modes * problem_.dims.n_omega, "occupancy buckets"); if (count == 0 || count > occupancy_bucket_limit) throw std::invalid_argument("the occupancy needs between 1 and 128 buckets"); @@ -16912,9 +17062,9 @@ void GeneralWorkspace::separable_split_step(const double dz, // real space up to the transforms' rounding. const auto coarse = coarse_spectrum_; const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const double inverse_time = 1.0 / static_cast(problem_.n_t); + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); for (std::size_t unit = 0; unit < transverse_units(); unit += transverse_units_per_chunk()) { const auto chunk = transverse_chunk(unit); @@ -16940,11 +17090,11 @@ void GeneralWorkspace::separable_split_step(const double dz, const auto scratch = transverse_; const auto coarse = coarse_spectrum_; const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; + const auto n_pol = problem_.dims.n_pol_state; const auto layout = transverse_layout(); - const auto n = problem_.n_omega; + const auto n = problem_.dims.n_omega; const double inverse_nodes = inverse_transverse_count(); - const double inverse_time = 1.0 / static_cast(problem_.n_t); + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); launch::parallel_for( "solprop_native.free.split_input", Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), @@ -16998,7 +17148,7 @@ void GeneralWorkspace::form_split_factor(const double dz) { const auto factor = split_factor_; launch::parallel_for( "solprop_native.absorber.split_factor", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), KOKKOS_LAMBDA(const std::size_t index) { factor(index) = Kokkos::exp(-0.5 * rate(index) * dz); }); @@ -17022,10 +17172,10 @@ void GeneralWorkspace::split_step(const double dz, const ComplexView2& advanced) return; } const auto coarse = coarse_spectrum_; - const double inverse_n = 1.0 / static_cast(problem_.n_t); + const double inverse_n = 1.0 / static_cast(problem_.dims.n_t); launch::parallel_for( "solprop_native.absorber.split_pad", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { coarse(column, index) = inverse_n * advanced(column, index); }); @@ -17033,7 +17183,7 @@ void GeneralWorkspace::split_step(const double dz, const ComplexView2& advanced) const auto active = active_mask_; launch::parallel_for( "solprop_native.absorber.split_commit", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { advanced(column, index) = active(index) != 0U ? coarse(column, index) : Complex{}; @@ -17041,7 +17191,7 @@ void GeneralWorkspace::split_step(const double dz, const ComplexView2& advanced) } std::vector GeneralWorkspace::field_to_host() const { - std::vector result(problem_.n_columns() * problem_.n_omega); + std::vector result(problem_.n_columns() * problem_.dims.n_omega); field_to_host(result); return result; } @@ -17068,14 +17218,14 @@ void GeneralWorkspace::field_to_host(const std::span destination) const require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - require_extent(destination.size(), problem_.n_columns() * problem_.n_omega, + require_extent(destination.size(), problem_.n_columns() * problem_.dims.n_omega, "field destination"); execution_space_.fence("solprop_native field_to_host"); cross_to_host(current_, destination); } std::vector GeneralWorkspace::stage_to_host(const std::size_t stage) const { - std::vector result(problem_.n_columns() * problem_.n_omega); + std::vector result(problem_.n_columns() * problem_.dims.n_omega); stage_to_host(stage, result); return result; } @@ -17085,9 +17235,9 @@ void GeneralWorkspace::stage_to_host(const std::size_t stage, require_open(); if (!stages_ready_) throw std::logic_error("no trial stages have been evaluated"); - if (stage >= problem_.n_stages) + if (stage >= problem_.stepper.n_stages) throw std::out_of_range("stage index is outside the tableau"); - require_extent(destination.size(), problem_.n_columns() * problem_.n_omega, + require_extent(destination.size(), problem_.n_columns() * problem_.dims.n_omega, "stage destination"); execution_space_.fence("solprop_native stage_to_host"); const auto selected = Kokkos::subview(stages_, stage, Kokkos::ALL, Kokkos::ALL); @@ -17099,8 +17249,8 @@ void GeneralWorkspace::stage_to_host(const std::size_t stage, } const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, selected); for (std::size_t column = 0; column < problem_.n_columns(); ++column) - for (std::size_t index = 0; index < problem_.n_omega; ++index) - destination[column * problem_.n_omega + index] = host(column, index); + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) + destination[column * problem_.dims.n_omega + index] = host(column, index); } std::vector GeneralWorkspace::projected_to_host(const bool coarse) { @@ -17158,7 +17308,7 @@ std::map GeneralWorkspace::base_memory_ledger() const const auto real_bytes = sizeof(double); const auto byte_bytes = sizeof(unsigned char); const auto index_bytes = sizeof(std::size_t); - const auto field = problem_.n_columns() * problem_.n_omega * complex_bytes; + const auto field = problem_.n_columns() * problem_.dims.n_omega * complex_bytes; const auto modal = modal_real_.size() * real_bytes + modal_complex_.size() * complex_bytes; // Aliased views (free space, SEP 28) own no storage of their own. @@ -17173,7 +17323,7 @@ std::map GeneralWorkspace::base_memory_ledger() const projected_real_.size() * real_bytes + projected_complex_.size() * complex_bytes; return { {"held_current_host_derivative", - problem_.time_window_derivative.capacity() * real_bytes}, + problem_.grid.time_window_derivative.capacity() * real_bytes}, {"held_current", held_current_.size() * real_bytes}, {"gated_current_prefix", (gated_prefix_.size() + gated_filtered_.size()) * real_bytes}, @@ -17209,7 +17359,7 @@ std::map GeneralWorkspace::base_memory_ledger() const {"rk_dense", (rk_dense_coefficients_.size() + rk_dense_held_.size()) * complex_bytes + rk_dense_stiff_.size() * byte_bytes}, - {"noise", problem_.modified_noise ? field : 0}, + {"noise", problem_.stepper.modified_noise ? field : 0}, // The Runge-Kutta march's whole-step propagator, or the low-storage march's // table or step pair (solprop's `propagators` row). {"propagators", @@ -17226,7 +17376,8 @@ std::map GeneralWorkspace::base_memory_ledger() const {"node_field", node}, {"node_polarisation", polarisation}, {"projected", projected}, - {"projected_coarse", problem_.projection_coarse_matrix.empty() ? 0 : projected}, + {"projected_coarse", + problem_.transform.projection_coarse_matrix.empty() ? 0 : projected}, {"projected_spectrum", projected_spectrum_.size() * complex_bytes}, {"projected_coarse_spectrum", projected_coarse_spectrum_.size() * complex_bytes}, {"absorber_spectrum", absorber_spectrum_.size() * complex_bytes}, @@ -17241,49 +17392,55 @@ std::map GeneralWorkspace::base_memory_ledger() const // The native problem owns host copies as well as the execution-space views. // Count retained vector capacity, including construction-only coefficients. {"host_problem_arrays", - (problem_.envelope_loss_factor.capacity() + problem_.carrier.capacity() + - problem_.general_synthesis.capacity() + problem_.general_projection.capacity() + - problem_.general_frequency_factor.capacity() + - problem_.general_delayed_spectrum.capacity() + problem_.phi_terms.capacity() + - problem_.prefactor_terms.capacity() + problem_.spatial_on_axis.capacity() + - problem_.spatial_k_squared.capacity() + - problem_.spatial_k_squared_low.capacity() + - problem_.spatial_numerator.capacity() + problem_.spatial_clamped.capacity() + - problem_.spatial_kappa_sq.capacity()) * + (problem_.plasma.envelope_loss_factor.capacity() + + problem_.grid.carrier.capacity() + problem_.general.synthesis.capacity() + + problem_.general.projection.capacity() + + problem_.general.frequency_factor.capacity() + + problem_.general.delayed_spectrum.capacity() + + problem_.linear.phi_terms.capacity() + + problem_.linear.prefactor_terms.capacity() + + problem_.spatial.on_axis.capacity() + problem_.spatial.k_squared.capacity() + + problem_.spatial.k_squared_low.capacity() + + problem_.spatial.numerator.capacity() + problem_.spatial.clamped.capacity() + + problem_.spatial.kappa_sq.capacity()) * complex_bytes + - (problem_.envelope_rule_nodes.capacity() + - problem_.envelope_rule_weights.capacity() + - problem_.envelope_drude_factor.capacity() + - problem_.omega_window.capacity() + problem_.omega.capacity() + - problem_.power_weight.capacity() + problem_.time_window.capacity() + - problem_.absorber_fine.capacity() + problem_.absorber_coarse.capacity() + - problem_.spatial_absorber_matrix.capacity() + - problem_.spatial_spectral_weights.capacity() + - problem_.spatial_kperp_sq.capacity() + - problem_.spatial_kperp_sq_low.capacity() + - problem_.spatial_frame.capacity() + - problem_.spatial_spectral_rate.capacity() + - problem_.spatial_user_rate.capacity() + - problem_.spatial_absorber_rate.capacity() + - problem_.spatial_kperp_rate.capacity() + problem_.spatial_k_x.capacity() + - problem_.spatial_k_y.capacity() + problem_.synthesis_matrix.capacity() + - problem_.projection_matrix.capacity() + - problem_.projection_coarse_matrix.capacity() + - problem_.tableau_matrix.capacity() + problem_.tableau_nodes.capacity() + - problem_.solution_weights.capacity() + problem_.error_weights.capacity() + - problem_.raman_undamped.capacity() + - problem_.raman_tensor_weights.capacity() + - problem_.rate_coefficients.capacity() + - problem_.channel_parameters.capacity() + - problem_.channel_coefficients.capacity() + - problem_.raman_row_component.capacity() + - problem_.raman_component_fraction.capacity() + - problem_.general_tensor.capacity() + - problem_.general_delayed_tensor.capacity()) * + (problem_.plasma.envelope_rule_nodes.capacity() + + problem_.plasma.envelope_rule_weights.capacity() + + problem_.plasma.envelope_drude_factor.capacity() + + problem_.grid.omega_window.capacity() + problem_.grid.omega.capacity() + + problem_.grid.power_weight.capacity() + + problem_.grid.time_window.capacity() + problem_.absorber.fine.capacity() + + problem_.absorber.coarse.capacity() + + problem_.spatial.absorber_matrix.capacity() + + problem_.spatial.spectral_weights.capacity() + + problem_.spatial.kperp_sq.capacity() + + problem_.spatial.kperp_sq_low.capacity() + + problem_.spatial.frame.capacity() + + problem_.spatial.spectral_rate.capacity() + + problem_.spatial.user_rate.capacity() + + problem_.spatial.absorber_rate.capacity() + + problem_.spatial.kperp_rate.capacity() + problem_.spatial.k_x.capacity() + + problem_.spatial.k_y.capacity() + + problem_.transform.synthesis_matrix.capacity() + + problem_.transform.projection_matrix.capacity() + + problem_.transform.projection_coarse_matrix.capacity() + + problem_.stepper.tableau_matrix.capacity() + + problem_.stepper.tableau_nodes.capacity() + + problem_.stepper.solution_weights.capacity() + + problem_.stepper.error_weights.capacity() + + problem_.raman.undamped.capacity() + + problem_.raman.tensor_weights.capacity() + + problem_.plasma.rate_coefficients.capacity() + + problem_.plasma.channel_parameters.capacity() + + problem_.plasma.channel_coefficients.capacity() + + problem_.raman.row_component.capacity() + + problem_.raman.component_fraction.capacity() + + problem_.general.tensor.capacity() + + problem_.general.delayed_tensor.capacity()) * real_bytes + - problem_.general_node_response.capacity() * index_bytes + - problem_.crop_index.capacity() * index_bytes + - (problem_.active_mask.capacity() + problem_.norm_mask.capacity()) * + problem_.general.node_response.capacity() * index_bytes + + problem_.grid.crop_index.capacity() * index_bytes + + (problem_.grid.active_mask.capacity() + problem_.grid.norm_mask.capacity()) * byte_bytes}, {"scaled_transform_matrices", (scaled_synthesis_.size() + scaled_projection_.size() + @@ -17412,7 +17569,7 @@ std::map GeneralWorkspace::base_memory_ledger() const plasma_tile_totals_.size()) * real_bytes}, {"error_partials", 3 * problem_.n_columns() * reduction_slots_ * real_bytes}, - {"quadrature_partials", problem_.projection_coarse_matrix.empty() + {"quadrature_partials", problem_.transform.projection_coarse_matrix.empty() ? 0 : 2 * problem_.n_columns() * real_bytes}, {"weighted_control", @@ -17783,166 +17940,142 @@ void GeneralWorkspace::close() noexcept { raman_kernel_ready_ = false; } +template +decltype(auto) GeneralState::with_team(Function function, + Arguments&&... arguments) const { + const CpuTeamScope team(workspace_->cpu_team()); + // NOLINTNEXTLINE(readability-avoid-return-with-void-value): returns what the member returns, void included + return std::invoke(function, *workspace_, std::forward(arguments)...); +} + GeneralState::GeneralState(Problem problem) : workspace_(std::make_unique(std::move(problem))) {} GeneralState::~GeneralState() { close(); } void GeneralState::set_initial(const std::span field, const std::span noise) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_initial(field, noise); + with_team(&GeneralWorkspace::set_initial, field, noise); } std::vector GeneralState::trial_step(const double dz, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->trial_step(dz, scalars); + return with_team(&GeneralWorkspace::trial_step, dz, scalars); } void GeneralState::prepare_trial(const double dz, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->prepare_trial(dz, scalars); + with_team(&GeneralWorkspace::prepare_trial, dz, scalars); } std::vector GeneralState::adams_source(const std::vector& input, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_source(input, scalars); + return with_team(&GeneralWorkspace::adams_source, input, scalars); } void GeneralState::adopt(const std::vector& field, const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adopt(field, dz); + with_team(&GeneralWorkspace::adopt, field, dz); } double GeneralState::low_storage_step(const double dz, const TrialScalars& scalars, const bool rebuild_table) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->low_storage_step(dz, scalars, rebuild_table); + return with_team(&GeneralWorkspace::low_storage_step, dz, scalars, rebuild_table); } void GeneralState::adams_allocate(const std::size_t slots, const bool compensated, const bool interpolate) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_allocate(slots, compensated, interpolate); + with_team(&GeneralWorkspace::adams_allocate, slots, compensated, interpolate); } void GeneralState::adams_compensation_start(const std::size_t slot, const std::vector& rate) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_compensation_start(slot, rate); + with_team(&GeneralWorkspace::adams_compensation_start, slot, rate); } void GeneralState::adams_start(const std::size_t slot, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_start(slot, scalars); + with_team(&GeneralWorkspace::adams_start, slot, scalars); } double GeneralState::adams_magnitude() { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_magnitude(); + return with_team(&GeneralWorkspace::adams_magnitude); } void GeneralState::adams_scale(const double rtol, const double atol, const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_scale(rtol, atol, weights); + with_team(&GeneralWorkspace::adams_scale, rtol, atol, weights); } std::vector GeneralState::adams_fused_trial(const double dz, const AdamsWeights& weights, const std::vector& divisor, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_fused_trial(dz, weights, divisor, scalars); + return with_team(&GeneralWorkspace::adams_fused_trial, dz, weights, divisor, scalars); } void GeneralState::adams_fused_accept(const double dz, const AdamsWeights& keep, const std::size_t free, const AdamsWeights& dense) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_fused_accept(dz, keep, free, dense); -} -void GeneralState::adams_hold() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_hold(); -} -void GeneralState::adams_restore() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_restore(); + with_team(&GeneralWorkspace::adams_fused_accept, dz, keep, free, dense); } +void GeneralState::adams_hold() { with_team(&GeneralWorkspace::adams_hold); } +void GeneralState::adams_restore() { with_team(&GeneralWorkspace::adams_restore); } void GeneralState::table_upload(const TableKind kind, std::vector edges, const double lower, const double upper, const std::complex* coefficients, const std::size_t n_intervals, const std::size_t n_terms, const std::size_t n_values) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_upload(kind, std::move(edges), lower, upper, coefficients, - n_intervals, n_terms, n_values); + with_team(&GeneralWorkspace::table_upload, kind, std::move(edges), lower, upper, + coefficients, n_intervals, n_terms, n_values); } void GeneralState::table_evaluate(const std::vector& slots, const std::vector& positions) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_evaluate(slots, positions); + with_team(&GeneralWorkspace::table_evaluate, slots, positions); } void GeneralState::table_rate(const std::size_t slot, const double z, const bool before) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_rate(slot, z, before); + with_team(&GeneralWorkspace::table_rate, slot, z, before); } std::vector GeneralState::table_terms_to_host(const TableKind kind, const std::size_t slot) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->table_terms_to_host(kind, slot); + return with_team(&GeneralWorkspace::table_terms_to_host, kind, slot); } std::vector GeneralState::phase_sum_to_host(const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->phase_sum_to_host(weights); + return with_team(&GeneralWorkspace::phase_sum_to_host, weights); } std::vector GeneralState::adams_row_names() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_row_names(); + return with_team(&GeneralWorkspace::adams_row_names); } std::array GeneralState::adams_row_extents(const std::string& name) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_row_extents(name); + return with_team(&GeneralWorkspace::adams_row_extents, name); } std::vector GeneralState::adams_rows_to_host(const std::string& name) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_rows_to_host(name); + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.adams_rows_to_host(name); }); } void GeneralState::adams_rows_to_host(const std::string& name, std::complex* destination, const std::size_t size) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_rows_to_host(name, destination, size); + with_team([&](GeneralWorkspace& workspace) { + workspace.adams_rows_to_host(name, destination, size); + }); } void GeneralState::adams_rows_from_host(const std::string& name, const std::vector& values) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_rows_from_host(name, values); + with_team(&GeneralWorkspace::adams_rows_from_host, name, values); } void GeneralState::adams_observe(const AdamsWeights& weights, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_observe(weights, scalars); + with_team(&GeneralWorkspace::adams_observe, weights, scalars); } std::vector GeneralState::adams_exponential_trial( const double dz, const ExponentialWeights& weights, const std::vector& divisor, const TrialScalars& step, const TrialScalars& sample, const std::vector& rate) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_exponential_trial(dz, weights, divisor, step, sample, rate); + return with_team(&GeneralWorkspace::adams_exponential_trial, dz, weights, divisor, + step, sample, rate); } void GeneralState::adams_exponential_accept(const double dz, const std::size_t free, const AdamsWeights& dense) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_exponential_accept(dz, free, dense); + with_team(&GeneralWorkspace::adams_exponential_accept, dz, free, dense); } void GeneralState::rk_dense_allocate(const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_allocate(weights); + with_team(&GeneralWorkspace::rk_dense_allocate, weights); } void GeneralState::rk_dense_hold(const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_hold(dz); + with_team(&GeneralWorkspace::rk_dense_hold, dz); } void GeneralState::rk_dense_observe(const double theta, const TrialScalars& carry) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_observe(theta, carry); + with_team(&GeneralWorkspace::rk_dense_observe, theta, carry); } void GeneralState::rk_dense_restore() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_restore(); + with_team(&GeneralWorkspace::rk_dense_restore); } void GeneralState::rk_dense_release() noexcept { if (workspace_) @@ -17950,41 +18083,33 @@ void GeneralState::rk_dense_release() noexcept { } void GeneralState::adams_exponential_observe(const double xi, const double dz, const ExponentialWeights& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_exponential_observe(xi, dz, weights); + with_team(&GeneralWorkspace::adams_exponential_observe, xi, dz, weights); } std::vector GeneralState::weighted_error(const std::vector& scale) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->weighted_error(scale); + return with_team(&GeneralWorkspace::weighted_error, scale); } void GeneralState::set_flow_actions(const std::vector& forward, const std::vector& inverse, const std::vector& identity) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_flow_actions(forward, inverse, identity); + with_team(&GeneralWorkspace::set_flow_actions, forward, inverse, identity); } std::vector GeneralState::flow_actions_to_host(const bool inverse) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->flow_actions_to_host(inverse); + return with_team(&GeneralWorkspace::flow_actions_to_host, inverse); } std::vector GeneralState::adams_block_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_block_to_host(); + return with_team(&GeneralWorkspace::adams_block_to_host); } std::vector GeneralState::adams_exponent_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_exponent_to_host(); + return with_team(&GeneralWorkspace::adams_exponent_to_host); } std::vector GeneralState::form_flow_actions(const std::size_t section, const double dz, const std::vector& fractions) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->form_flow_actions(section, dz, fractions); + return with_team(&GeneralWorkspace::form_flow_actions, section, dz, fractions); } void GeneralState::prepare_raman(const std::vector& rates, const std::vector& densities) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->prepare_raman(rates, densities); + with_team(&GeneralWorkspace::prepare_raman, rates, densities); } std::vector GeneralState::statistics(const std::uint64_t flags, const std::vector& windows, @@ -17993,104 +18118,83 @@ GeneralState::statistics(const std::uint64_t flags, const std::vector& w const double intensity_factor, const std::vector& channel_preionisation, const std::vector& power_weight) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->statistics(flags, windows, on_axis, density, taper, preionisation, - intensity_factor, channel_preionisation, power_weight); + return with_team(&GeneralWorkspace::statistics, flags, windows, on_axis, density, + taper, preionisation, intensity_factor, channel_preionisation, + power_weight); } std::vector GeneralState::whitened_statistics(const std::vector& root, const std::uint64_t flags, const std::vector& windows, const double intensity_factor, const bool spectral) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->whitened_statistics(root, flags, windows, intensity_factor, - spectral); + return with_team(&GeneralWorkspace::whitened_statistics, root, flags, windows, + intensity_factor, spectral); } std::vector GeneralState::spatial_statistics(const std::uint64_t flags, const double intensity_factor, const double z) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->spatial_statistics(flags, intensity_factor, z); + return with_team(&GeneralWorkspace::spatial_statistics, flags, intensity_factor, z); } std::vector GeneralState::plane_fluence_trace( const double center_x, const double center_y, const double dx, const double dy, const double extent, const double spacing, const std::size_t n_radii, const std::vector& cosines, const std::vector& sines) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->plane_fluence_trace(center_x, center_y, dx, dy, extent, spacing, - n_radii, cosines, sines); + return with_team(&GeneralWorkspace::plane_fluence_trace, center_x, center_y, dx, dy, + extent, spacing, n_radii, cosines, sines); } std::vector GeneralState::axis_spectrum(const std::vector& matrix) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->axis_spectrum(matrix); + return with_team(&GeneralWorkspace::axis_spectrum, matrix); } std::vector GeneralState::axis_populations(const std::vector& matrix, const std::vector& channel_preionisation) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->axis_populations(matrix, channel_preionisation); + return with_team(&GeneralWorkspace::axis_populations, matrix, channel_preionisation); } std::vector GeneralState::aperture_statistics( const std::uint64_t flags, const std::vector& apertures, const int exponent, const std::vector& regions, const double intensity_factor) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->aperture_statistics(flags, apertures, exponent, regions, - intensity_factor); -} -void GeneralState::finish(const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->finish(dz); + return with_team(&GeneralWorkspace::aperture_statistics, flags, apertures, exponent, + regions, intensity_factor); } +void GeneralState::finish(const double dz) { with_team(&GeneralWorkspace::finish, dz); } void GeneralState::bind_monitor(const std::vector& weights, const std::vector& column_weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_monitor(weights, column_weights); + with_team(&GeneralWorkspace::bind_monitor, weights, column_weights); } void GeneralState::unbind_monitor() noexcept { if (workspace_) workspace_->unbind_monitor(); } std::vector GeneralState::monitor_reading() { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->monitor_reading(); + return with_team(&GeneralWorkspace::monitor_reading); } void GeneralState::bind_edge(const std::vector& faces, const std::vector& clip, const std::vector& totals, const std::vector& synthesis, const std::size_t first_row) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_edge(faces, clip, totals, synthesis, first_row); + with_team(&GeneralWorkspace::bind_edge, faces, clip, totals, synthesis, first_row); } CrossedBytes GeneralState::crossed_bytes() const noexcept { return workspace_->crossed_bytes(); } void GeneralState::bind_edge_lines(const EdgeLines& lines) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_edge_lines(lines); + with_team(&GeneralWorkspace::bind_edge_lines, lines); } EdgeNumbers GeneralState::edge_reading(const bool faces, const bool whole, const bool totals, const bool strip, const bool current) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->edge_reading(faces, whole, totals, strip, current); + return with_team(&GeneralWorkspace::edge_reading, faces, whole, totals, strip, + current); } void GeneralState::bind_occupancy(const std::vector& buckets, const std::size_t count) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_occupancy(buckets, count); -} -void GeneralState::promote_fsal() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->promote_fsal(); -} -void GeneralState::clear_fsal() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->clear_fsal(); + with_team(&GeneralWorkspace::bind_occupancy, buckets, count); } +void GeneralState::promote_fsal() { with_team(&GeneralWorkspace::promote_fsal); } +void GeneralState::clear_fsal() { with_team(&GeneralWorkspace::clear_fsal); } void GeneralState::set_absorber_quiet(const bool quiet) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_absorber_quiet(quiet); + with_team(&GeneralWorkspace::set_absorber_quiet, quiet); } double GeneralState::quadrature_error() const { return workspace_ ? workspace_->quadrature_error() @@ -18108,54 +18212,49 @@ std::string GeneralState::projection_provider() const { } std::string GeneralState::gemm_shape_policy() const { const CpuTeamScope team(workspace_->cpu_team()); - return GeneralWorkspace::gemm_shape_policy(); + return std::string(GeneralWorkspace::gemm_shape_policy()); } std::size_t GeneralState::response_team_size() const noexcept { return workspace_ ? workspace_->response_team_size() : 0; } std::vector GeneralState::field_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->field_to_host(); + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.field_to_host(); }); } void GeneralState::set_crossing_workers(const std::size_t workers) noexcept { if (workspace_) workspace_->set_crossing_workers(workers); } void GeneralState::field_to_host(const std::span destination) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->field_to_host(destination); + with_team([&](GeneralWorkspace& workspace) { workspace.field_to_host(destination); }); } const Complex* GeneralState::host_field() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->host_field(); + return with_team(&GeneralWorkspace::host_field); } void GeneralState::count_to_host(const std::size_t bytes) const noexcept { workspace_->count_to_host(bytes); } void GeneralState::stage_to_host(const std::size_t stage, const std::span destination) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->stage_to_host(stage, destination); + with_team([&](GeneralWorkspace& workspace) { + workspace.stage_to_host(stage, destination); + }); } std::vector GeneralState::stage_to_host(const std::size_t stage) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->stage_to_host(stage); + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.stage_to_host(stage); }); } std::vector GeneralState::projected_to_host(const bool coarse) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->projected_to_host(coarse); + return with_team(&GeneralWorkspace::projected_to_host, coarse); } std::map GeneralState::dispersive_path() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->dispersive_path(); + return with_team(&GeneralWorkspace::dispersive_path); } std::map GeneralState::memory_ledger() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->memory_ledger(); + return with_team(&GeneralWorkspace::memory_ledger); } CpuSizing GeneralState::execution_sizing() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->execution_sizing(); + return with_team(&GeneralWorkspace::execution_sizing); } const CpuExecution& GeneralState::cpu_execution() const { return workspace_->cpu_execution(); diff --git a/native/src/general_modal.cpp b/native/src/general_modal.cpp index e9342573f..e28272146 100644 --- a/native/src/general_modal.cpp +++ b/native/src/general_modal.cpp @@ -3,8 +3,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/general_modal.hpp" + +#include "solprop_native/dispatch.hpp" #include "solprop_native/fft_plan.hpp" #include "solprop_native/launch.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -30,7 +33,7 @@ constexpr std::size_t max_components = 3; // before its vectoriser runs, and its early complete unrolling leaves such a nest alone // unless asked: the loop then ran a sample at a time, packing pairs through the stack // (a failed store-to-load forward per sample), at about 60 % of a general modal run. -// Clang and icpx unroll the nest themselves and device builds keep their own unrolling, +// Clang unrolls the nest itself and device builds keep their own unrolling, // so the macro is empty there. Unrolling moves no rounding. #if defined(__CUDACC__) || defined(__HIPCC__) || defined(__clang__) #define SOLPROP_NATIVE_UNROLL @@ -65,9 +68,7 @@ std::size_t product(const std::size_t left, const std::size_t right, const char* void require_size(const std::size_t actual, const std::size_t expected, const char* name) { if (actual != expected) - throw std::invalid_argument(std::string("general modal ") + name + " has " + - std::to_string(actual) + " values, expected " + - std::to_string(expected)); + refuse("general modal ", name, " has ", actual, " values, expected ", expected); } template bool finite(const std::vector& values) { @@ -134,8 +135,8 @@ static_assert(symmetric_monomials(3) == 10 && symmetric_monomial(3, 2, 0, 1) == symmetric_monomial(2, 1, 0, 1) == 2); bool general_crop_is_contiguous(const Problem& problem) { - const auto& crop = problem.crop_index; - if (crop.size() != problem.n_omega || crop.empty()) + const auto& crop = problem.grid.crop_index; + if (crop.size() != problem.dims.n_omega || crop.empty()) return false; for (std::size_t omega = 0; omega < crop.size(); ++omega) if (crop[omega] != crop.front() + omega) @@ -158,31 +159,32 @@ int general_team_lanes() { } // namespace void require_general_modal(const Problem& problem) { - if (!problem.general_modal) { - if (!problem.general_synthesis.empty() || !problem.general_projection.empty() || - !problem.general_tensor.empty() || !problem.general_frequency_factor.empty() || - !problem.general_delayed_tensor.empty() || - !problem.general_delayed_spectrum.empty() || - !problem.general_node_response.empty()) + if (!problem.general.enabled) { + if (!problem.general.synthesis.empty() || !problem.general.projection.empty() || + !problem.general.tensor.empty() || !problem.general.frequency_factor.empty() || + !problem.general.delayed_tensor.empty() || + !problem.general.delayed_spectrum.empty() || + !problem.general.node_response.empty()) throw std::invalid_argument("general modal arrays require general_modal"); return; } - const auto components = problem.general_components; - const auto nodes = problem.general_nodes; + const auto components = problem.general.components; + const auto nodes = problem.general.nodes; if (components == 0 || components > max_components) throw std::invalid_argument("the general modal route resolves one to three " "Cartesian components"); - if (nodes == 0 || problem.general_node_chunk == 0 || - problem.general_node_chunk > nodes) + if (nodes == 0 || problem.general.node_chunk == 0 || + problem.general.node_chunk > nodes) throw std::invalid_argument("general_node_chunk must lie in [1, general_nodes]"); - if (problem.family != FieldFamily::field_real || problem.n_pol_state != 1 || - problem.n_pol_transform != 1 || problem.separable_geometry()) + if (problem.dims.family != FieldFamily::field_real || problem.dims.n_pol_state != 1 || + problem.dims.n_pol_transform != 1 || problem.separable_geometry()) throw std::invalid_argument("the general modal route runs one channel axis on a " "field grid with no polarisation axis"); - if (problem.nonlinear || problem.plasma || problem.raman || problem.envelope_plasma) + if (problem.kerr.nonlinear || problem.plasma.enabled || problem.raman.enabled || + problem.plasma.envelope) throw std::invalid_argument("the general modal route carries its own regional " "responses and no legacy nonlinear term"); - if (problem.n_to == 0 || problem.n_to % 2 != 0) + if (problem.dims.n_to == 0 || problem.dims.n_to % 2 != 0) throw std::invalid_argument("the general modal route needs an even fine grid"); // A field grid's coarse bins are the leading bins of the fine spectrum (solprop // `fourier.crop_index`), so the source reads and writes them as one contiguous @@ -190,41 +192,42 @@ void require_general_modal(const Problem& problem) { if (!general_crop_is_contiguous(problem)) throw std::invalid_argument("the general modal route needs the coarse bins to be " "one contiguous block of the fine spectrum"); - if (!std::isfinite(problem.general_pad_scale) || problem.general_pad_scale <= 0.0 || - !std::isfinite(problem.general_epsilon0) || problem.general_epsilon0 <= 0.0) + if (!std::isfinite(problem.general.pad_scale) || problem.general.pad_scale <= 0.0 || + !std::isfinite(problem.general.epsilon0) || problem.general.epsilon0 <= 0.0) throw std::invalid_argument( "general modal pad scale and vacuum permittivity must be finite and positive"); - if (problem.general_map_intervals == 0 || problem.general_map_n_deg == 0) + if (problem.general.map_intervals == 0 || problem.general.map_n_deg == 0) throw std::invalid_argument("general modal maps carry at least one section of at " "least one Chebyshev term along z"); const auto map_terms = - product(problem.general_map_intervals, problem.general_map_n_deg, "map terms"); - if (map_terms > 1 && problem.n_stages == 0) + product(problem.general.map_intervals, problem.general.map_n_deg, "map terms"); + if (map_terms > 1 && problem.stepper.n_stages == 0) throw std::invalid_argument("moving general modal maps need at least one stage"); const auto rows = product(nodes, components, "node rows"); - const auto maps = - product(product(rows, problem.n_modes, "maps"), problem.n_omega, "maps"); - require_size(problem.general_synthesis.size(), product(map_terms, maps, "map terms"), + const auto maps = product(product(rows, problem.dims.n_modes, "maps"), + problem.dims.n_omega, "maps"); + require_size(problem.general.synthesis.size(), product(map_terms, maps, "map terms"), "synthesis"); - require_size(problem.general_projection.size(), product(map_terms, maps, "map terms"), + require_size(problem.general.projection.size(), product(map_terms, maps, "map terms"), "projection"); if (map_terms > 1) - static_cast(product(product(problem.n_stages, map_terms, "map weights"), - sizeof(double), "map weight bytes")); - require_size(problem.general_tensor.size(), + static_cast( + product(product(problem.stepper.n_stages, map_terms, "map weights"), + sizeof(double), "map weight bytes")); + require_size(problem.general.tensor.size(), product(nodes, fourth_power(components), "tensor"), "tensor"); - require_size(problem.general_frequency_factor.size(), problem.n_omega, + require_size(problem.general.frequency_factor.size(), problem.dims.n_omega, "frequency factor"); - if (!finite(problem.general_synthesis) || !finite(problem.general_projection) || - !finite(problem.general_tensor) || !finite(problem.general_frequency_factor)) + if (!finite(problem.general.synthesis) || !finite(problem.general.projection) || + !finite(problem.general.tensor) || !finite(problem.general.frequency_factor)) throw std::invalid_argument("general modal maps must be finite"); - const auto chunk_rows = product(problem.general_node_chunk, components, "chunk rows"); - static_cast(product(product(chunk_rows, problem.n_to, "chunk buffers"), + const auto chunk_rows = product(problem.general.node_chunk, components, "chunk rows"); + static_cast(product(product(chunk_rows, problem.dims.n_to, "chunk buffers"), sizeof(Complex), "chunk bytes")); - if (problem.general_terms == 0) { - if (problem.general_responses != 0 || !problem.general_delayed_tensor.empty() || - !problem.general_delayed_spectrum.empty() || - !problem.general_node_response.empty()) + if (problem.general.terms == 0) { + if (problem.general.responses != 0 || !problem.general.delayed_tensor.empty() || + !problem.general.delayed_spectrum.empty() || + !problem.general.node_response.empty()) throw std::invalid_argument("a general modal problem with no delayed term " "carries no delayed arrays"); return; @@ -232,44 +235,46 @@ void require_general_modal(const Problem& problem) { if (components != max_components) throw std::invalid_argument( "a delayed general modal response needs the three Cartesian components"); - if (problem.general_responses == 0) + if (problem.general.responses == 0) throw std::invalid_argument("a delayed general modal term needs a response"); const auto channels = - product(problem.general_responses, problem.general_terms, "delayed channels"); - require_size(problem.general_delayed_tensor.size(), + product(problem.general.responses, problem.general.terms, "delayed channels"); + require_size(problem.general.delayed_tensor.size(), product(channels, fourth_power(components), "delayed tensor"), "delayed tensor"); - require_size(problem.general_delayed_spectrum.size(), - product(channels, problem.n_to + 1, "delayed spectrum"), + require_size(problem.general.delayed_spectrum.size(), + product(channels, problem.dims.n_to + 1, "delayed spectrum"), "delayed spectrum"); - require_size(problem.general_node_response.size(), nodes, "node response"); - if (!finite(problem.general_delayed_tensor) || - !finite(problem.general_delayed_spectrum)) + require_size(problem.general.node_response.size(), nodes, "node response"); + if (!finite(problem.general.delayed_tensor) || + !finite(problem.general.delayed_spectrum)) throw std::invalid_argument("general modal delayed data must be finite"); - for (const auto owner : problem.general_node_response) - if (owner > problem.general_responses) + for (const auto owner : problem.general.node_response) + if (owner > problem.general.responses) throw std::invalid_argument("general modal node response names no response"); static_cast(product( - product(product(problem.general_node_chunk, general_pair_histories, "pairs"), - 2 * problem.n_to, "pair buffers"), + product(product(problem.general.node_chunk, general_pair_histories, "pairs"), + 2 * problem.dims.n_to, "pair buffers"), sizeof(Complex), "pair bytes")); } GeneralModalSource::GeneralModalSource(const Problem& problem) - : n_modes_(problem.n_modes), n_omega_(problem.n_omega), n_to_(problem.n_to), - components_(problem.general_components), nodes_(problem.general_nodes), - chunk_(problem.general_node_chunk), terms_(problem.general_terms), - map_terms_(problem.general_map_intervals * problem.general_map_n_deg), - map_n_deg_(problem.general_map_n_deg), n_stages_(problem.n_stages), - crop_offset_(problem.crop_index.empty() ? 0 : problem.crop_index.front()), - real_form_(problem.general_real_form), epsilon0_(problem.general_epsilon0), + : n_modes_(problem.dims.n_modes), n_omega_(problem.dims.n_omega), + n_to_(problem.dims.n_to), components_(problem.general.components), + nodes_(problem.general.nodes), chunk_(problem.general.node_chunk), + terms_(problem.general.terms), + map_terms_(problem.general.map_intervals * problem.general.map_n_deg), + map_n_deg_(problem.general.map_n_deg), n_stages_(problem.stepper.n_stages), + crop_offset_(problem.grid.crop_index.empty() ? 0 + : problem.grid.crop_index.front()), + real_form_(problem.general.real_form), epsilon0_(problem.general.epsilon0), kernel_pool_(CpuPool::kernels, problem.cpu_execution), fft_pool_(CpuPool::fft, problem.cpu_execution), raman_pool_(CpuPool::raman_fft, problem.cpu_execution), raman_tail_pool_(CpuPool::raman_tail_fft, problem.cpu_execution) { require_general_modal(problem); require_cpu_execution(problem.cpu_execution); - if (!problem.general_modal) + if (!problem.general.enabled) throw std::invalid_argument("the problem does not declare the general modal route"); kernels_ = kernel_pool_.get(CpuOperation::stage, n_modes_ * n_omega_); response_ = @@ -285,22 +290,21 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) // A gas region's dispersive channels respond where every regional tensor may be // zero; with no regional tensor and no delayed response the contraction is // identically zero, and it and its transform are skipped. - const bool tensor = - std::any_of(problem.general_tensor.begin(), problem.general_tensor.end(), - [](const double value) { return value != 0.0; }); + const bool tensor = std::ranges::any_of( + problem.general.tensor, [](const double value) { return value != 0.0; }); regional_ = tensor || terms_ != 0; - responds_ = tensor || problem.dispersive; + responds_ = tensor || problem.dispersive.enabled; const auto chunk_rows = chunk_ * components_; if (moving()) { // A moving guide holds every coefficient of both maps and one evaluated map of // each, which `combine` rewrites per stage; the weights and their host copy are // allocated here so that a trial allocates nothing. - const auto one_map = problem.general_synthesis.size() / map_terms_; + const auto one_map = problem.general.synthesis.size() / map_terms_; maps_ = kernel_pool_.get(CpuOperation::modal, 2 * one_map * map_n_deg_); synthesis_terms_ = ComplexView("solprop_native.general.synthesis_terms", - problem.general_synthesis.size()); + problem.general.synthesis.size()); projection_terms_ = ComplexView("solprop_native.general.projection_terms", - problem.general_projection.size()); + problem.general.projection.size()); synthesis_ = ComplexView("solprop_native.general.stage_synthesis", one_map); projection_ = ComplexView("solprop_native.general.stage_projection", one_map); map_weights_ = @@ -309,18 +313,19 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) stage_sections_.assign(n_stages_, 0); } else { synthesis_ = ComplexView("solprop_native.general.synthesis", - problem.general_synthesis.size()); + problem.general.synthesis.size()); projection_ = ComplexView("solprop_native.general.projection", - problem.general_projection.size()); + problem.general.projection.size()); } frequency_factor_ = ComplexView("solprop_native.general.frequency_factor", n_omega_); - tensor_ = RealView("solprop_native.general.tensor", problem.general_tensor.size()); + tensor_ = RealView("solprop_native.general.tensor", problem.general.tensor.size()); scaled_ = ComplexView2("solprop_native.general.scaled_state", n_modes_, n_omega_); source_ = ComplexView2("solprop_native.general.source", n_modes_, n_omega_); node_spectrum_ = ComplexView2("solprop_native.general.node_spectrum", chunk_rows, n_to_); node_field_ = ComplexView2("solprop_native.general.node_field", chunk_rows, n_to_); - real_signal_ = general_real_signal_enabled && real_form_ && !problem.dispersive; + real_signal_ = + general_real_signal_enabled && real_form_ && !problem.dispersive.enabled; if (real_signal_) { half_spectrum_ = AliasComplexView2(node_spectrum_.data(), chunk_rows, n_to_ / 2 + 1); @@ -334,9 +339,9 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) if (terms_ != 0) { const auto pair_rows = chunk_ * general_pair_histories; delayed_tensor_ = RealView("solprop_native.general.delayed_tensor", - problem.general_delayed_tensor.size()); + problem.general.delayed_tensor.size()); delayed_spectrum_ = ComplexView("solprop_native.general.delayed_spectrum", - problem.general_delayed_spectrum.size()); + problem.general.delayed_spectrum.size()); node_response_ = IndexView("solprop_native.general.node_response", nodes_); node_drive_ = RealView2("solprop_native.general.node_drive", pair_rows, 2 * n_to_); node_drive_spectrum_ = ComplexView2("solprop_native.general.node_drive_spectrum", @@ -346,19 +351,20 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) node_histories_ = RealView2("solprop_native.general.node_histories", pair_rows, 2 * n_to_); node_delayed_ = RealView2("solprop_native.general.node_delayed", chunk_rows, n_to_); - upload(kernels_, delayed_tensor_, problem.general_delayed_tensor); - upload(kernels_, delayed_spectrum_, problem.general_delayed_spectrum); - upload(kernels_, node_response_, problem.general_node_response); + upload(kernels_, delayed_tensor_, problem.general.delayed_tensor); + upload(kernels_, delayed_spectrum_, problem.general.delayed_spectrum); + upload(kernels_, node_response_, problem.general.node_response); } - upload(kernels_, moving() ? synthesis_terms_ : synthesis_, problem.general_synthesis); + upload(kernels_, moving() ? synthesis_terms_ : synthesis_, problem.general.synthesis); upload(kernels_, moving() ? projection_terms_ : projection_, - problem.general_projection); - upload(kernels_, frequency_factor_, problem.general_frequency_factor); - upload(kernels_, tensor_, problem.general_tensor); - if (problem.dispersive) + problem.general.projection); + upload(kernels_, frequency_factor_, problem.general.frequency_factor); + upload(kernels_, tensor_, problem.general.tensor); + if (problem.dispersive.enabled) dispersive_ = std::make_unique( problem, kernel_pool_, - GeneralDispersive::StageViews{node_spectrum_, node_field_, node_source_}); + GeneralDispersive::StageViews{ + .spectrum = node_spectrum_, .field = node_field_, .source = node_source_}); kernels_.fence("solprop_native general modal upload"); } @@ -500,8 +506,9 @@ void GeneralModalSource::evaluate() { else graph::execute(*forward_plan_, node_polarisation_, node_source_, KokkosFFT::Normalization::none); - } else if (!live) + } else if (!live) { graph::deep_copy(kernels_, node_source_, Complex{}); + } // The products' spectra join the regional polarisation's, or are the chunk's. if (live) dispersive_->combine(!regional_); @@ -743,12 +750,13 @@ void GeneralModalSource::respond(const std::size_t first, const std::size_t coun respond_components<2, true>(first, count); else respond_components<1, true>(first, count); - } else if (components_ == 3) + } else if (components_ == 3) { respond_components<3, false>(first, count); - else if (components_ == 2) + } else if (components_ == 2) { respond_components<2, false>(first, count); - else + } else { respond_components<1, false>(first, count); + } } template @@ -834,7 +842,7 @@ void GeneralModalSource::respond_components(const std::size_t first, for (std::size_t i = 0; i < components; ++i) polarisation(row + i, time) = epsilon0 * values[i]; }); - } else + } else { for (std::size_t i = 0; i < components; ++i) { Kokkos::parallel_for( Kokkos::ThreadVectorRange(team, samples), [=](const int sample) { @@ -860,6 +868,7 @@ void GeneralModalSource::respond_components(const std::size_t first, polarisation(row + i, time) = (epsilon0 * value).real(); }); } + } if (!delayed) return; // The delayed response's pair drive: the real field's `E_k E_l`, or the @@ -981,34 +990,9 @@ void GeneralModalSource::project(const std::size_t first, const std::size_t coun #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) project_device(first, count); #else - switch (projection_lanes_) { - case 2048: - project_lanes<2048>(first, count); - break; - case 1024: - project_lanes<1024>(first, count); - break; - case 512: - project_lanes<512>(first, count); - break; - case 256: - project_lanes<256>(first, count); - break; - case 128: - project_lanes<128>(first, count); - break; - case 64: - project_lanes<64>(first, count); - break; - case 32: - project_lanes<32>(first, count); - break; - case 16: - project_lanes<16>(first, count); - break; - default: - project_lanes(first, count); - } + dispatch_among<2048, 1024, 512, 256, 128, 64, 32, 16>( + projection_lanes_, [&] { project_lanes(first, count); }, + [&] { project_lanes(first, count); }); #endif } diff --git a/native/src/helpers.cpp b/native/src/helpers.cpp index b181d624c..cabb4f2c4 100644 --- a/native/src/helpers.cpp +++ b/native/src/helpers.cpp @@ -6,6 +6,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/vmath.hpp" #include @@ -40,7 +41,7 @@ template using Output = nb::ndarray, nb::c_contig, nb::device::cpu>; std::string shape_text(const std::size_t rows, const std::size_t columns) { - return "(" + std::to_string(rows) + ", " + std::to_string(columns) + ")"; + return concat("(", rows, ", ", columns, ")"); } template @@ -53,16 +54,14 @@ std::ptrdiff_t clenshaw_points(const Input& z, const Input throw std::invalid_argument( "clenshaw_points: edges needs at least two breakpoints"); if (c.shape(0) != n_edges - 1 || c.shape(1) < 1) - throw std::invalid_argument("clenshaw_points: coefficients of shape " + - shape_text(c.shape(0), c.shape(1)) + - "[...] do not match " + std::to_string(n_edges) + - " breakpoints with at least one term"); + refuse("clenshaw_points: coefficients of shape ", + shape_text(c.shape(0), c.shape(1)), "[...] do not match ", n_edges, + " breakpoints with at least one term"); const std::size_t n_points = z.shape(0); const std::size_t n_values = c.shape(2); if (out.shape(0) != n_points || out.shape(1) != n_values) - throw std::invalid_argument("clenshaw_points: out has shape " + - shape_text(out.shape(0), out.shape(1)) + ", expected " + - shape_text(n_points, n_values)); + refuse("clenshaw_points: out has shape ", shape_text(out.shape(0), out.shape(1)), + ", expected ", shape_text(n_points, n_values)); return chebyshev::clenshaw_points(z.data(), n_points, edges.data(), n_edges, lower, upper, c.data(), c.shape(1), n_values, out.data()); } @@ -72,14 +71,11 @@ void basis_rows(const Input& points, const double upper, const Output& weights) { const std::size_t n = points.shape(0); if (n > lagrange::max_nodes) - throw std::invalid_argument("basis_rows: " + std::to_string(n) + - " nodes exceed the " + - std::to_string(lagrange::max_nodes) + " supported"); + refuse("basis_rows: ", n, " nodes exceed the ", lagrange::max_nodes, " supported"); if (rows.shape(0) != n || rows.shape(1) != n || weights.shape(0) != n) - throw std::invalid_argument( - "basis_rows: rows " + shape_text(rows.shape(0), rows.shape(1)) + - " and weights (" + std::to_string(weights.shape(0)) + - ",) must be (n, n) and (n,) for n = " + std::to_string(n) + " nodes"); + refuse("basis_rows: rows ", shape_text(rows.shape(0), rows.shape(1)), + " and weights (", weights.shape(0), ",) must be (n, n) and (n,) for n = ", n, + " nodes"); lagrange::basis_rows(points.data(), n, upper, integrate, rows.data(), weights.data()); } @@ -91,16 +87,13 @@ void trial_tables(const Input& nodes, const std::size_t order, const Output& endpoint, const Output& defects) { const std::size_t n = nodes.shape(0); const std::size_t n_orders = orders.size(); - const std::size_t highest = - orders.empty() ? 0 : *std::max_element(orders.begin(), orders.end()); - const bool scored = std::all_of(orders.begin(), orders.end(), - [](const std::size_t value) { return value > 0; }); + const std::size_t highest = orders.empty() ? 0 : *std::ranges::max_element(orders); + const bool scored = + std::ranges::all_of(orders, [](const std::size_t value) { return value > 0; }); if (n_orders == 0 || n_orders > 3 || !scored || order == 0 || order > highest || highest > n || highest + 1 > lagrange::max_nodes) - throw std::invalid_argument("trial_tables: order " + std::to_string(order) + - " and " + std::to_string(n_orders) + - " scored orders up to " + std::to_string(highest) + - " do not fit " + std::to_string(n) + " nodes"); + refuse("trial_tables: order ", order, " and ", n_orders, " scored orders up to ", + highest, " do not fit ", n, " nodes"); const std::size_t width = highest + 1; const std::size_t carried_size = compensated ? order : 0; if (predictor.shape(0) != order || predictor.shape(1) != order || @@ -111,11 +104,10 @@ void trial_tables(const Input& nodes, const std::size_t order, endpoint.shape(0) != (compensated ? order + 1 : 0) || defects.shape(0) != (compensated ? n_orders : 0) || defects.shape(1) != (compensated ? width : 0)) - throw std::invalid_argument( - "trial_tables: the tables must be (order, order), (width, width) and " - "(n_orders, 2 width), and the compensation's (width - 1,), (order,), " - "(order + 1,) and (n_orders, width) when compensated, else empty; order " + - std::to_string(order) + ", width " + std::to_string(width)); + refuse("trial_tables: the tables must be (order, order), (width, width) and " + "(n_orders, 2 width), and the compensation's (width - 1,), (order,), " + "(order + 1,) and (n_orders, width) when compensated, else empty; order ", + order, ", width ", width); lagrange::trial_tables(nodes.data(), order, orders.data(), n_orders, width, compensated, predictor.data(), corrector.data(), factored.data(), spans.data(), carried.data(), endpoint.data(), @@ -125,9 +117,7 @@ void trial_tables(const Input& nodes, const std::size_t order, void require_length(const char* name, const std::size_t expected, const std::size_t actual) { if (actual != expected) - throw std::invalid_argument(std::string(name) + ": the arrays have lengths " + - std::to_string(expected) + " and " + - std::to_string(actual)); + refuse(name, ": the arrays have lengths ", expected, " and ", actual); } void vector_exp(const Input& x, const Output& out) { diff --git a/native/src/low_storage.cpp b/native/src/low_storage.cpp index e9998449e..fad1ea646 100644 --- a/native/src/low_storage.cpp +++ b/native/src/low_storage.cpp @@ -4,6 +4,8 @@ #include "solprop_native/low_storage.hpp" +#include "solprop_native/adams_march.hpp" + #include #include #include @@ -11,8 +13,6 @@ #include #include -#include "solprop_native/adams_march.hpp" - // As in adams_march.cpp (solprop #135): the step's positions and the table keys round // as solprop's Python does, so this file takes no contraction (CMake) and the optimize // attribute this pragma gives every function below keeps GCC from inlining one into a @@ -60,8 +60,8 @@ void put(std::vector& phi_delta, const std::size_t row, const std::size_ const std::vector& weights) { if (weights.size() != n_phi) throw std::logic_error("an increment's weights are not the stage rows' width"); - std::copy(weights.begin(), weights.end(), - phi_delta.begin() + static_cast(row * n_phi)); + std::ranges::copy(weights, + phi_delta.begin() + static_cast(row * n_phi)); } } // namespace @@ -103,7 +103,7 @@ double schedule_key(const std::vector& keys, const std::vector& // `searchsorted(edges, z + 0.5 * dz) - 1`, clamped to the steps. const double half = 0.5 * dz; const double midpoint = z + half; - const auto found = std::lower_bound(edges.begin(), edges.end(), midpoint); + const auto found = std::ranges::lower_bound(edges, midpoint); const auto index = static_cast(found - edges.begin()) - 1; const auto last = static_cast(keys.size()) - 1; return keys[static_cast( diff --git a/native/src/raman_fft.cpp b/native/src/raman_fft.cpp index bf15c01a8..f062d8c84 100644 --- a/native/src/raman_fft.cpp +++ b/native/src/raman_fft.cpp @@ -4,8 +4,6 @@ #include "solprop_native/raman_fft.hpp" -#include - #include #include #include @@ -15,6 +13,8 @@ #include #include +#include + namespace solprop_native { struct AccelerateRamanPlan::Implementation { @@ -52,8 +52,8 @@ struct AccelerateRamanPlan::Implementation { const auto last = plan.rows * (task + 1) / plan.task_count; auto* input = plan.source + static_cast(first) * plan.source_stride; auto* output = plan.target + static_cast(first) * plan.target_stride; - DSPDoubleSplitComplex source_split{input, input + 1}; - DSPDoubleSplitComplex target_split{output, output + 1}; + DSPDoubleSplitComplex source_split{.realp = input, .imagp = input + 1}; + DSPDoubleSplitComplex target_split{.realp = output, .imagp = output + 1}; const auto log_length = static_cast(std::countr_zero(plan.samples)); if (plan.forward) { vDSP_fftm_zropD(plan.setups[task].get(), &source_split, 2, plan.source_stride, diff --git a/native/src/rk_march.cpp b/native/src/rk_march.cpp index 301bdfa47..359593807 100644 --- a/native/src/rk_march.cpp +++ b/native/src/rk_march.cpp @@ -4,6 +4,8 @@ #include "solprop_native/rk_march.hpp" +#include "solprop_native/core.hpp" + #include #include #include @@ -11,8 +13,6 @@ #include #include -#include "solprop_native/core.hpp" - // As in adams_march.cpp (solprop #135): the optimize attribute this pragma gives every // function below keeps GCC from inlining it into a contracting caller under link-time // optimisation. Clang carries the file's setting on each instruction. @@ -170,7 +170,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const double target = bounds.target; const int ulps = policy.landings.landing_ulps; std::uint64_t accepted = 0; - const auto elapsed = [&]() { + const auto elapsed = [&] { return std::chrono::duration(Clock::now() - started).count(); }; const auto record = [&](const double end, const double dz, const double norm, @@ -192,7 +192,7 @@ Result run(const Policy& policy, State& state, Engine& engine, }; // The step's boundary check, as the Adams march takes it (SEP 45 4.8); true when the // call returns. - const auto watched = [&]() { + const auto watched = [&] { if (!bounds.watch) return false; const auto& rules = bounds.watch_policy; @@ -201,7 +201,7 @@ Result run(const Policy& policy, State& state, Engine& engine, // A session's owning monitor checks the step itself (SEP 47 section 6.4). try { verdict = watch::owned_check( - *bounds.owner, [&engine]() { return engine.reading(); }, state.z, + *bounds.owner, [&engine] { return engine.reading(); }, state.z, bounds.replay, schedule.steps, state.cursor); } catch (const std::exception& error) { fail(result, march::Stage::accept, error); @@ -266,7 +266,7 @@ Result run(const Policy& policy, State& state, Engine& engine, }; // A stop, after the attempt's commit and accounting (SEP 47 section 5.5), the // call's own bound, and the readings' buffer; true when the call returns. - const auto bounded = [&]() { + const auto bounded = [&] { if (march::stop_requested(bounds)) { result.reason = march::Reason::stopped; return true; @@ -291,7 +291,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const std::size_t n_steps = schedule.steps.size(); if (schedule.edges.size() != n_steps + 1) throw std::invalid_argument("a schedule has one edge more than its steps"); - const auto running = [&]() { + const auto running = [&] { return state.cursor < n_steps && state.z < target - bounds.slack; }; // A stop already requested ends the call before its first attempt. @@ -381,6 +381,7 @@ Result run(const Policy& policy, State& state, Engine& engine, } state.proposal = next; } else { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) if (dense && !march::reached(target, end, ulps)) { if (!bounds.extend) { // solprop records the step's extension from the state before it, then diff --git a/native/src/session.cpp b/native/src/session.cpp index b5045c6c6..b5af2ce02 100644 --- a/native/src/session.cpp +++ b/native/src/session.cpp @@ -4,6 +4,9 @@ #include "solprop_native/session.hpp" +#include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" + #include #include #include @@ -13,8 +16,6 @@ #include #include -#include "solprop_native/core.hpp" - // As in adams_march.cpp (solprop #135): the scheduler's comparisons and the export's // recipe round as solprop's Python does, so this file takes no contraction (CMake) and // the optimize attribute this pragma gives every function below keeps GCC from @@ -206,7 +207,7 @@ Session::Session(std::unique_ptr resident, Method method, Plan plan, "a session interpolates its planes with the march's dense output"); dense_ = runge_kutta.dense.get(); } - if (!std::is_sorted(plan_.planes.begin(), plan_.planes.end())) + if (!std::ranges::is_sorted(plan_.planes)) throw std::invalid_argument("a session's save planes increase"); if (plan_.next_plane > plan_.planes.size()) throw std::invalid_argument("a session's first plane is past its planes"); @@ -445,7 +446,7 @@ void Session::progress(Batch& batch) { event.kind = EventKind::progress; event.z = here(); drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::snapshot(Batch& batch) { @@ -459,7 +460,7 @@ void Session::snapshot(Batch& batch) { event.ticket = true; event.generation = generation_; drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::terminal(Batch& batch, const Outcome outcome, Failure failure) { @@ -473,7 +474,7 @@ void Session::terminal(Batch& batch, const Outcome outcome, Failure failure) { event.ticket = outcome != Outcome::completed && !poisoned(); event.generation = generation_; drain(batch); - batch.event = event; + batch.event = std::move(event); } Batch Session::issue(Batch batch, const Outcome outcome) { @@ -515,7 +516,7 @@ void Session::warning(Batch& batch, const Notice& notice) { event.generation = generation_; event.notice = notice; drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::monitor_stop(Batch& batch, const Notice& notice) { @@ -581,7 +582,7 @@ bool Session::attach(Batch& batch) { // `DriverSession._attach_monitor` after the launch plane's commit: restore a resumed // record, then record and judge the launch (`attach_monitor`). attached_ = true; - const auto& plan = *plan_.monitor; + const auto& plan = plan_.monitor.value(); if (plan.resume) { // `BoundaryMonitor.resume` continues the edge's record first. if (edge_) @@ -691,15 +692,15 @@ Batch Session::advance(const std::size_t work_limit, const double deadline) { if (state_.load() == State::finished) throw LifecycleError("the session has ended and cannot advance"); const auto started = Clock::now(); - const auto elapsed = [&started]() { + const auto elapsed = [&started] { return std::chrono::duration(Clock::now() - started).count(); }; const std::size_t budget = std::max(work_limit, 1); const std::uint64_t before = attempts_; Batch batch; - const auto made = [&]() { return static_cast(attempts_ - before); }; + const auto made = [&] { return static_cast(attempts_ - before); }; // Issue a filled batch, terminal or not. - const auto ready = [&]() { + const auto ready = [&] { const Outcome outcome = batch.event.kind == EventKind::terminal ? batch.event.outcome : Outcome::none; return issue(std::move(batch), outcome); @@ -807,7 +808,7 @@ Batch Session::advance(const std::size_t work_limit, const double deadline) { target, landing_ulps_) : 0.0; const std::size_t n_steps = schedule().steps.size(); - const auto running = [&]() { + const auto running = [&] { // `replay_to`'s and `march_to`'s loops: to the edge within the slack, or // reached. const double z = position(); @@ -1000,9 +1001,9 @@ void Session::acknowledge(const std::uint64_t sequence) { const std::scoped_lock lock(mutex_); require_open(); if (!pending_ || sequence != sequence_) - throw LifecycleError("receipt " + std::to_string(sequence) + - " is not the pending batch's: a foreign, stale or repeated " - "acknowledgment"); + refuse("receipt ", sequence, + " is not the pending batch's: a foreign, stale or repeated " + "acknowledgment"); pending_ = false; state_.store(outcome_.load() != Outcome::none ? State::finished : State::ready, std::memory_order_release); @@ -1057,11 +1058,11 @@ SnapshotReceipt Session::request_snapshot() { std::uint64_t value = latch_.load(std::memory_order_acquire); while (true) { if ((value & 1U) != 0) - return {value >> 1U, true}; + return {.request = value >> 1U, .coalesced = true}; const std::uint64_t next = (((value >> 1U) + 1) << 1U) | 1U; if (latch_.compare_exchange_weak(value, next, std::memory_order_acq_rel)) { halt_.store(true, std::memory_order_release); - return {next >> 1U, false}; + return {.request = next >> 1U, .coalesced = false}; } } } @@ -1119,10 +1120,12 @@ bool Session::close() noexcept { runge_kutta->dense.reset(); } resident_.reset(); - history_ = {}; - row_z_ = {}; - row_step_ = {}; - row_values_ = {}; + // Move-assigning an empty vector releases the storage and cannot throw; assigning + // `{}` goes through the initializer-list assignment, which may. + history_ = std::vector(); + row_z_ = std::vector(); + row_step_ = std::vector(); + row_values_ = std::vector(); return true; } diff --git a/native/src/session_edge.cpp b/native/src/session_edge.cpp index 2cb9fd8d8..563f67797 100644 --- a/native/src/session_edge.cpp +++ b/native/src/session_edge.cpp @@ -70,7 +70,7 @@ bool Session::edge_launch(Batch& batch) { const transverse::Reading reading = edge_reading(true, !resumed, edge_->every_step() && resumed); // The input noise's readings, on a resumed launch too: the reads judge above them. - const auto& noise = plan_.monitor->edge_noise; + const auto& noise = plan_.monitor.value().edge_noise; return edge_judged( batch, edge_->launch(reading, position(), nullptr, noise ? &*noise : nullptr), Continuation::edge_launch); diff --git a/native/src/session_statistics.cpp b/native/src/session_statistics.cpp index d739bb788..986dded29 100644 --- a/native/src/session_statistics.cpp +++ b/native/src/session_statistics.cpp @@ -2,11 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/statistics_plan.hpp" - #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" #include "solprop_native/far_field.hpp" +#include "solprop_native/refuse.hpp" +#include "solprop_native/statistics_plan.hpp" #include #include @@ -82,9 +82,9 @@ class Wanted { bool operator()(const Stat stat) const { return set_[static_cast(stat)]; } - bool any(std::initializer_list stats) const { - return std::any_of(stats.begin(), stats.end(), - [this](const Stat stat) { return (*this)(stat); }); + [[nodiscard]] bool any(std::initializer_list stats) const { + return std::ranges::any_of(stats, + [this](const Stat stat) { return (*this)(stat); }); } private: @@ -199,14 +199,13 @@ void species(const Plan& plan, const double z, const double taper, if (weights.size() != n_nodes) throw std::invalid_argument("species statistics require one weight per node"); const auto valid = [](const std::vector& values) { - return std::all_of(values.begin(), values.end(), [](const double value) { + return std::ranges::all_of(values, [](const double value) { return std::isfinite(value) && value >= 0.0; }); }; if (fractions.empty() || !valid(densities) || !valid(fractions) || !valid(peaks) || !valid(potentials) || !valid(weights) || - std::any_of(fractions.begin(), fractions.end(), - [](const double value) { return value > 1.0; })) + std::ranges::any_of(fractions, [](const double value) { return value > 1.0; })) throw std::invalid_argument("species statistics require finite non-negative " "populations and fractions in [0, 1]"); std::vector electrons(n_species * n_nodes); @@ -637,7 +636,7 @@ template void Table::validate(const char* name) const { check(n_terms >= 1 && n_values >= 1, std::string("table has no terms: ") + name); check(coefficients.size() == (edges.size() - 1) * n_terms * n_values, std::string("table's coefficients have the wrong extent: ") + name); - check(std::is_sorted(edges.begin(), edges.end()), + check(std::ranges::is_sorted(edges), std::string("table's edges do not increase: ") + name); check(lower <= edges.front() && upper >= edges.back(), std::string("table's span is not inside its slack: ") + name); @@ -651,9 +650,8 @@ template std::vector Table::at(const double z) const chebyshev::clenshaw_points(&z, 1, edges.data(), edges.size(), lower, upper, coefficients.data(), n_terms, n_values, out.data()); if (refused >= 0) - throw std::out_of_range( - "z = " + std::to_string(z) + " m is outside a statistics table's span [" + - std::to_string(edges.front()) + ", " + std::to_string(edges.back()) + "] m"); + refuse("z = ", z, " m is outside a statistics table's span [", + edges.front(), ", ", edges.back(), "] m"); return out; } @@ -729,17 +727,16 @@ void Plan::validate() const { "lacks the on-axis synthesis its axis row reads"); check(channel_preionisation.size() == n_channels, "lacks the preionisation its axis row reads"); - check(axis_node < static_cast(n_nodes), - "has an axis node past the nodes"); + check(std::cmp_less(axis_node, n_nodes), "has an axis node past the nodes"); check(std::isfinite(pad_scale) && pad_scale > 0.0, "has a pad scale that is not positive"); } if (wanted(Stat::zdw)) - check(zdw.size() == n_modes && std::all_of(zdw.begin(), zdw.end(), - [](const RealTable& table) { - return table.present() && - table.n_values == 1; - }), + check(zdw.size() == n_modes && std::ranges::all_of(zdw, + [](const RealTable& table) { + return table.present() && + table.n_values == 1; + }), "lacks a mode's zero-dispersion wavelength table"); if (wanted.any({Stat::beam_radius, Stat::fwhm_r})) { check(route == Route::spatial && widths != Widths::none, @@ -856,8 +853,8 @@ std::vector evaluate(GeneralState& state, const Plan& plan, const double for (const Stat stat : plan.outputs) { const auto& values = results[static_cast(stat)]; if (values.size() != plan.width(stat)) - throw std::logic_error("statistic " + std::to_string(static_cast(stat)) + - " is not evaluated on this route"); + refuse("statistic ", static_cast(stat), + " is not evaluated on this route"); packed.insert(packed.end(), values.begin(), values.end()); } return packed; diff --git a/native/src/session_widths.cpp b/native/src/session_widths.cpp index 642c94619..80318cf9f 100644 --- a/native/src/session_widths.cpp +++ b/native/src/session_widths.cpp @@ -17,6 +17,7 @@ #endif #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/statistics_plan.hpp" #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -60,11 +62,11 @@ std::string scientific(const double value) { /** `solprop.backends.spatial._negative_fluence`. */ std::invalid_argument negative_fluence(const double value, const std::size_t node) { - return std::invalid_argument( + return std::invalid_argument(concat( "beam_radius and fwhm_r need a nonnegative fluence profile, and the signed " - "axial flux is " + - scientific(value) + " J/m^2 at node " + std::to_string(node) + - ", beyond its rounding budget; the width is undefined for it"); + "axial flux is ", + scientific(value), " J/m^2 at node ", node, + ", beyond its rounding budget; the width is undefined for it")); } /** `fluence_budget`'s one value: eight rounding units a term of the largest @@ -84,7 +86,7 @@ double interp(const double point, const std::vector& x, if (point > x.back()) return samples.back(); // The last `j` with `x[j] <= point`. - const auto above = std::upper_bound(x.begin(), x.end(), point); + const auto above = std::ranges::upper_bound(x, point); const auto j = static_cast(above - x.begin()) - 1; if (j == n - 1 || x[j] == point) return samples[j]; @@ -119,10 +121,9 @@ std::array line_widths(const std::vector& x, const std::vector& values, const double e2_fraction) { const auto finite = [](const double value) { return std::isfinite(value); }; - if (x.size() < 2 || values.size() != x.size() || - !std::all_of(x.begin(), x.end(), finite) || - !std::all_of(values.begin(), values.end(), finite) || - std::any_of(values.begin(), values.end(), [](const double v) { return v < 0.0; })) + if (x.size() < 2 || values.size() != x.size() || !std::ranges::all_of(x, finite) || + !std::ranges::all_of(values, finite) || + std::ranges::any_of(values, [](const double v) { return v < 0.0; })) throw std::invalid_argument("Cartesian fluence widths require finite axes " "and matching nonnegative samples"); require_uniform(x); @@ -215,7 +216,7 @@ std::array radial(const double* radius, const double* fluence, throw std::invalid_argument("radial fluence moments are not representable"); if (energy <= 0.0) return undefined(); - const double second = std::sqrt(2.0) * std::sqrt(moment / energy); + const double second = std::numbers::sqrt2 * std::sqrt(moment / energy); double largest = fluence[0]; for (std::size_t index = 1; index < n; ++index) largest = std::max(largest, fluence[index]); diff --git a/native/src/solprop_native/__init__.py b/native/src/solprop_native/__init__.py index 4fcdf28bc..d25d92c76 100644 --- a/native/src/solprop_native/__init__.py +++ b/native/src/solprop_native/__init__.py @@ -292,7 +292,9 @@ class BuildInfo: ``"scalar libm"``, the system's, on Arm and in device builds (``SOLPROP_NATIVE_VECTOR_MATH``). gemm_provider, tpl_blas_enabled, blas_backend, blas_thread_policy - Dense-transform implementation and external-library threading policy. + Dense-transform implementation (always ``"HYBRID"``: the vendor GEMM for an + eligible transform, the custom kernels for the rest) and external-library + threading policy. blas_selection How configuration chose the BLAS: ``automatic: ...``, ``explicit: ...`` or ``distribution: ...`` or ``disabled: ...``, including the native-order GEMM @@ -631,9 +633,9 @@ def check_build() -> None: "the build enables OpenMP but its Kokkos execution space is " f"{info.execution_space!r}" ) - # A CPU build runs OpenMP, or Kokkos Threads where it must add no OpenMP runtime. + # A CPU build runs OpenMP. expected_execution = { - "none": {"OpenMP", "Threads"}, + "none": {"OpenMP"}, "cuda": {"Cuda"}, "hip": {"HIP"}, } @@ -646,11 +648,8 @@ def check_build() -> None: raise BackendError( f"the {info.device_backend} build has no visible accelerator device" ) - providers = {"CUSTOM", "KOKKOSBLAS_NATIVE", "KOKKOSBLAS_TPL", "HYBRID"} - if info.gemm_provider not in providers: + if info.gemm_provider != "HYBRID": raise BackendError(f"unknown compiled GEMM provider {info.gemm_provider!r}") - if info.gemm_provider == "KOKKOSBLAS_TPL" and not info.tpl_blas_enabled: - raise BackendError("the TPL GEMM provider was built without a BLAS TPL") if info.tpl_blas_enabled and info.blas_backend == "none": raise BackendError("the build enables a BLAS TPL but identifies no provider") if info.build_profile not in {"local", "distribution", "reference"}: diff --git a/native/src/solprop_native/_core.pyi b/native/src/solprop_native/_core.pyi index ad73940bb..02fd2a3e6 100644 --- a/native/src/solprop_native/_core.pyi +++ b/native/src/solprop_native/_core.pyi @@ -4,13 +4,244 @@ from __future__ import annotations +import enum from typing import Any, overload import numpy as np import numpy.typing as npt +class FieldFamily(enum.Enum): + field_real = 0 + + field_analytic = 1 + + envelope = 2 + +class Problem: + def __init__(self) -> None: ... + class Dims: + def __init__(self) -> None: ... + n_omega: int + n_t: int + n_to: int + n_modes: int + n_pol_state: int + n_pol_transform: int + n_nodes: int + node_chunk: int + family: FieldFamily + + class Grid: + def __init__(self) -> None: ... + crop_index: npt.NDArray[np.uint64] + active_mask: npt.NDArray[np.bool_] + norm_mask: npt.NDArray[np.bool_] + omega_window: npt.NDArray[np.float64] + omega: npt.NDArray[np.float64] + power_weight: npt.NDArray[np.float64] + time_window: npt.NDArray[np.float64] + time_window_derivative: npt.NDArray[np.float64] + carrier: npt.NDArray[np.complex128] + dt_fine: float + dt: float + + class Spatial: + def __init__(self) -> None: ... + n_x: int + n_y: int + k_squared: npt.NDArray[np.complex128] + k_squared_low: npt.NDArray[np.complex128] + numerator: npt.NDArray[np.complex128] + clamped: npt.NDArray[np.complex128] + kperp_sq: npt.NDArray[np.float64] + kperp_sq_low: npt.NDArray[np.float64] + frame: npt.NDArray[np.float64] + spectral_rate: npt.NDArray[np.float64] + user_rate: npt.NDArray[np.float64] + absorber_rate: npt.NDArray[np.float64] + kperp_rate: npt.NDArray[np.float64] + evanescent_rate: float + k_x: npt.NDArray[np.float64] + k_y: npt.NDArray[np.float64] + far_field_slots: int + n_p: int + kappa_sq: npt.NDArray[np.complex128] + absorber_matrix: npt.NDArray[np.float64] + spectral_weights: npt.NDArray[np.float64] + on_axis: npt.NDArray[np.complex128] + + class Flux: + def __init__(self) -> None: ... + k_squared: npt.NDArray[np.complex128] + k_squared_low: npt.NDArray[np.complex128] + kperp_sq: npt.NDArray[np.float64] + kperp_sq_low: npt.NDArray[np.float64] + table: bool + closure: bool + + class Stepper: + def __init__(self) -> None: ... + n_stages: int + tableau_matrix: npt.NDArray[np.float64] + tableau_nodes: npt.NDArray[np.float64] + solution_weights: npt.NDArray[np.float64] + error_weights: npt.NDArray[np.float64] + first_same_as_last: bool + weighted_controller: bool + modified_noise: bool + low_storage: bool + low_storage_stage_register: bool + low_storage_flows: int + low_storage_tiled_input: bool + + class Linear: + def __init__(self) -> None: ... + n_phi_terms: int + n_prefactor_terms: int + phi_table: bool + prefactor_table: bool + phi_terms: npt.NDArray[np.complex128] + prefactor_terms: npt.NDArray[np.complex128] + + class Absorber: + def __init__(self) -> None: ... + in_stage: bool + split_step: bool + fine: npt.NDArray[np.float64] + coarse: npt.NDArray[np.float64] + + class Transform: + def __init__(self) -> None: ... + synthesis_matrix: npt.NDArray[np.float64] + projection_matrix: npt.NDArray[np.float64] + projection_coarse_matrix: npt.NDArray[np.float64] + synthesis: float + projection: float + + class Kerr: + def __init__(self) -> None: ... + nonlinear: bool + coefficient: float + self_factor: float + conjugate_factor: float + harmonic_factor: float + take_real: bool + + class Plasma: + def __init__(self) -> None: ... + enabled: bool + envelope: bool + gated: bool + stash: bool + envelope_rule_nodes: npt.NDArray[np.float64] + envelope_rule_weights: npt.NDArray[np.float64] + envelope_drude_factor: npt.NDArray[np.float64] + envelope_loss_factor: npt.NDArray[np.complex128] + charge_mass_ratio: float + ionisation_potential: float + log_field_min: float + d_log_field: float + field_min: float + field_max: float + rate_max: float + rate_intervals: int + rate_coefficients: npt.NDArray[np.float64] + n_channels: int + band: bool + taps: npt.NDArray[np.float64] + channel_parameters: npt.NDArray[np.float64] + channel_coefficients: npt.NDArray[np.float64] + + class Raman: + def __init__(self) -> None: ... + enabled: bool + thg: bool + tensor_capable: bool + n_terms: int + fraction: float + epsilon0: float + undamped: npt.NDArray[np.float64] + tensor_weights: npt.NDArray[np.float64] + n_components: int + row_component: npt.NDArray[np.float64] + component_fraction: npt.NDArray[np.float64] + + class General: + def __init__(self) -> None: ... + enabled: bool + real_form: bool + map_intervals: int + map_n_deg: int + components: int + nodes: int + node_chunk: int + responses: int + terms: int + pad_scale: float + epsilon0: float + synthesis: npt.NDArray[np.complex128] + projection: npt.NDArray[np.complex128] + frequency_factor: npt.NDArray[np.complex128] + delayed_spectrum: npt.NDArray[np.complex128] + tensor: npt.NDArray[np.float64] + delayed_tensor: npt.NDArray[np.float64] + node_response: npt.NDArray[np.uint64] + + class Flow: + def __init__(self) -> None: ... + enabled: bool + active: int + sections: int + terms: int + slot: npt.NDArray[np.uint64] + frozen: npt.NDArray[np.complex128] + residual_terms: npt.NDArray[np.complex128] + left: npt.NDArray[np.complex128] + right: npt.NDArray[np.complex128] + eigenvalues: npt.NDArray[np.complex128] + route: npt.NDArray[np.uint64] + fallback_vectors: npt.NDArray[np.complex128] + fallback_triangular: npt.NDArray[np.complex128] + + class Dispersive: + def __init__(self) -> None: ... + enabled: bool + symmetric: bool + form: int + inputs: npt.NDArray[np.float64] + term_multipliers: npt.NDArray[np.float64] + coefficients: npt.NDArray[np.float64] + pair_constants: npt.NDArray[np.float64] + quartics: npt.NDArray[np.float64] + input_scale: npt.NDArray[np.float64] + output_scale: npt.NDArray[np.float64] + term_rows: npt.NDArray[np.uint64] + term_channels: npt.NDArray[np.uint64] + group_offsets: npt.NDArray[np.uint64] + group_members: npt.NDArray[np.uint64] + group_keys: npt.NDArray[np.uint64] + node_region: npt.NDArray[np.uint64] + + dims: Problem.Dims + grid: Problem.Grid + spatial: Problem.Spatial + flux: Problem.Flux + stepper: Problem.Stepper + linear: Problem.Linear + absorber: Problem.Absorber + transform: Problem.Transform + kerr: Problem.Kerr + plasma: Problem.Plasma + raman: Problem.Raman + general: Problem.General + flow: Problem.Flow + dispersive: Problem.Dispersive + cpu_execution: dict[str, int | bool] + fft_planning: dict[str, object] + def validate(self) -> None: ... + class State: - # The payload is `backend._native_problem` of a validated + # The problem is `backend._lower` of a validated # `solprop.translate.problem.PropagationProblem`: its shared grid, operator, # response and source components, with the route variant the adapter visited # (solprop SEP 47 section 6.2); `n_modes` and `n_pol_state` are the state's @@ -18,7 +249,7 @@ class State: # `(1, n_modes, n_pol, n_omega)`: `set_initial`, `field_to_host` and the # Adams rows carry the leading axis of one, which the adapter adds and removes, # and a single trajectory crosses the Python seam without it. - def __init__(self, problem: dict[str, object]) -> None: ... + def __init__(self, problem: Problem) -> None: ... def set_initial( self, field: npt.NDArray[np.complex128], diff --git a/native/src/solprop_native/backend.py b/native/src/solprop_native/backend.py index 23444ac36..b6e365a2e 100644 --- a/native/src/solprop_native/backend.py +++ b/native/src/solprop_native/backend.py @@ -410,20 +410,17 @@ def __init__( self._execution_source = ( _package.EXECUTION_SOURCE if cpu_execution is None else "given" ) - self._native_state = _core.State( - _native_problem( - problem, - request, - self._terms, - self._flow, - self._z_tables, - node_chunk=self._node_chunk, - general_chunk=self._general_chunk, - ) - | _low_storage_payload(problem, stepper, device=_on_device()) - | {"cpu_execution": (cpu_execution or _package.EXECUTION).native()} - | _planning_payload(fft_planning, stored) - ) + plan = _LowerPlan( + terms=self._terms, + flow=self._flow, + tables=self._z_tables, + node_chunk=self._node_chunk, + general_chunk=self._general_chunk, + cpu_execution=cpu_execution, + fft_planning=_planning(fft_planning, stored), + device=_on_device(), + ) + self._native_state = _core.State(_lower(problem, request, plan)) self._applied_execution = ExecutionRecord.from_native( self._native_state.cpu_execution ) @@ -435,18 +432,15 @@ def __init__( stored |= _keep_plans(self._native_state, wisdom) self._native_state.close() self._native_state = _core.State( - _native_problem( + _lower( problem, request, - self._terms, - self._flow, - self._z_tables, - node_chunk=self._node_chunk, - general_chunk=self._general_chunk, + dataclasses.replace( + plan, + cpu_execution=refined[0], + fft_planning=_planning(fft_planning, stored), + ), ) - | _low_storage_payload(problem, stepper, device=_on_device()) - | {"cpu_execution": refined[0].native()} - | _planning_payload(fft_planning, stored) ) self._execution_source = refined[1] self._applied_execution = ExecutionRecord.from_native( @@ -2190,28 +2184,20 @@ def native_node_chunk( return chunk -def _low_storage_payload( +def _low_storage_flows( problem: PropagationProblem, stepper: StepperSettings, *, device: bool -) -> dict[str, object]: - """Return the native problem's low-storage rows (solprop kernel schema 32). +) -> int | None: + """Return the low-storage march's flow volumes, or None for another march. - The march's own allocation: its registers in place of the Runge-Kutta views, the - stage register elided where solprop's estimate elides it, and the flows solprop's - estimate counts -- except that a device build recomputes under ``"auto"``, where - the estimate's CPU count over-states it by the table. A device's input crosses the - state's layout through tiles (``docs/dev/benchmarks.md``); a CPU's has its own. + The flows solprop's estimate counts (solprop kernel schema 32) -- except that a + device build recomputes under ``"auto"``, where the estimate's CPU count + over-states it by the table. A device's input crosses the state's layout through + tiles (``docs/dev/benchmarks.md``); a CPU's has its own. """ if not isinstance(stepper.integrator, LowStorageRungeKutta): - return {} + return None flows = low_storage_flows(problem, stepper, "native") - if device and stepper.propagator_mode == "auto": - flows = 0 - return { - "low_storage": True, - "low_storage_stage_register": low_storage_stage_register(problem, "native"), - "low_storage_flows": flows, - "low_storage_tiled_input": device, - } + return 0 if device and stepper.propagator_mode == "auto" else flows _MARCH_COUNTERS = ( @@ -2649,8 +2635,9 @@ def __init__(self, backend: NativeBackend, method: LowStorageRungeKutta) -> None self.atomic = True problem = backend._problem # pyright: ignore[reportPrivateUsage] - the owner's problem stepper = backend._stepper # pyright: ignore[reportPrivateUsage] - the owner's policy - payload = _low_storage_payload(problem, stepper, device=_on_device()) - self._flows = int(cast(int, payload["low_storage_flows"])) + flows = _low_storage_flows(problem, stepper, device=_on_device()) + assert flows is not None # the march is the low-storage one + self._flows = flows schedule = stepper.schedule assert schedule is not None # the translation refused the march without one self._keys, self._edges = table_steps(schedule), schedule.edges @@ -3093,47 +3080,6 @@ def _flow_allowance(modes: int) -> float: return 1.0 + 64.0 * modes * float(np.finfo(float).eps) -def _flow_payload(flow: _HostFlow | None) -> dict[str, object]: - """Pack a coupled flow's bins and residual data; its actions follow per step.""" - if flow is None: - return {"coupled_flow": False} - slot = np.zeros(flow.active.size, dtype=np.uint64) - slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) - residual = flow.residual_terms - empty = np.empty(0, dtype=np.complex128) - # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major - # (section, bin) order and carry their Schur factors; the rest take the - # eigenbasis, whose `W` and `Y` are zero where it is not used. - fallback = ~flow.route - route = np.zeros(flow.route.shape, dtype=np.uint64) - route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) - # The residual's data are zero on the bins the error norm does not score, so the - # engine's residual there is an exact zero and the frozen flow alone carries them, - # as in solprop's reference. Every term is finite, so each - # product with a zero is a signed zero and the running sums stay +0: the source is - # what adding nothing leaves, but for the sign of a zero. - scored = flow.scored[None, :, None, None] - return { - "coupled_flow": True, - "flow_active": int(np.count_nonzero(flow.active)), - "flow_sections": int(flow.frozen.shape[0]), - "flow_terms": 0 if residual is None else int(residual.shape[0]), - "flow_slot": slot, - "flow_frozen": empty - if residual is None - else np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)), - "flow_residual_terms": empty - if residual is None - else np.ascontiguousarray(np.where(scored, residual, 0.0)), - "flow_left": flow.left, - "flow_right": flow.right, - "flow_eigenvalues": flow.eigenvalues, - "flow_route": route.reshape(-1), - "flow_fallback_vectors": np.ascontiguousarray(flow.vectors[fallback]), - "flow_fallback_triangular": np.ascontiguousarray(flow.triangular[fallback]), - } - - _APERTURE_CONSTANTS = 11 """The constants before an aperture row's scale row (``far_field.hpp``).""" @@ -3212,21 +3158,22 @@ def _refined_record( return None -def _planning_payload( +def _planning( given: Mapping[str, object] | None, stored: Mapping[str, str] -) -> dict[str, object]: - """Return the problem's ``fft_planning``: the one given, or the run's own. +) -> dict[str, object] | None: + """Return the problem's FFT planning: the one given, or the run's own. A run's own is the profile's, or the plan policy's, offered the plans runs measured before (`stored`) and keeping those it measures (solprop ADR-076); one given, a - calibration's, plans on its own. + calibration's, plans on its own. None on a device, which plans with its vendor's + FFT. """ if _on_device(): - return {} + return None if given is not None: - return {"fft_planning": dict(given)} + return dict(given) chosen = _package.PLANNING or {"mode": "legacy"} - return {"fft_planning": dict(chosen) | {"stored": dict(stored), "keep": True}} + return dict(chosen) | {"stored": dict(stored), "keep": True} def _keep_plans(state: _core.State, path: Path | None) -> dict[str, str]: @@ -3286,144 +3233,320 @@ def _planning_record(state: _core.State) -> dict[str, object]: } -def _native_problem( - problem: PropagationProblem, - request: RunRequest, - fixed: StageTerms, - flow: _HostFlow | None = None, - tables: _ZTables | None = None, - *, - node_chunk: int | None = None, - general_chunk: int | None = None, -) -> dict[str, object]: +@dataclasses.dataclass(frozen=True) +class _LowerPlan: + """The backend's choices a lowering takes beside the problem and its request. + + `terms` are the fixed stage terms, `flow` a coupled flow's host factors and + `tables` the z tables; the chunks are the backend's lowered execution choices + (solprop SEP 47 section 6.5), the automatic plan's when absent. `cpu_execution` + is the workspace's CPU record (the process's when absent) and `fft_planning` its + FFT planning record (none on a device); `device` says the state is a device's. + """ + + terms: StageTerms + flow: _HostFlow | None = None + tables: _ZTables | None = None + node_chunk: int | None = None + general_chunk: int | None = None + cpu_execution: ExecutionRecord | None = None + fft_planning: Mapping[str, object] | None = None + device: bool = False + + +def _lower( + problem: PropagationProblem, request: RunRequest, plan: _LowerPlan +) -> _core.Problem: + """Fill the native problem's structs from solprop's problem, request and `plan`.""" stepper = request.stepper - # The constructor hands its lowered chunks; a direct caller takes the automatic - # plan's (solprop SEP 47 section 6.5). + node_chunk = plan.node_chunk if node_chunk is None: node_chunk = native_node_chunk( problem, plan_node_chunk(problem), device=_on_device() ) + general_chunk = plan.general_chunk if general_chunk is None and problem.transform.modal_tables is not None: general_chunk = general_planned_chunk(problem, plan_node_chunk(problem)) + native = _core.Problem() + native.cpu_execution = (plan.cpu_execution or _package.EXECUTION).native() + if plan.fft_planning is not None: + native.fft_planning = dict(plan.fft_planning) kerr = problem.responses.kerr_coefficients dispersive = problem.responses.dispersive_kerr + absorber = problem.responses.temporal_absorber + absorber_fine = problem.responses.temporal_absorber_fine + spatial_absorber = radial_absorber_matrix( + problem.transform, problem.grids.pad_scale + ) + temporal_in_stage = ( + absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" + ) + geometry = problem.transform.geometry + # Free space and schema 15's hybrid planar kind share the separable operator and + # the transverse FFT engine; the planar kind adds its slab rule in x. + free = geometry is not None and geometry.kind in ("free", "slab") + spatial_absorption = bool(spatial_absorber.size) or ( + free and geometry is not None and bool(np.any(geometry.absorber_rate)) + ) + + dims = native.dims + dims.n_omega = problem.shape[-1] + dims.n_t = problem.grids.n_t + dims.n_to = problem.grids.n_to + dims.n_modes = problem.shape[0] + dims.n_pol_state = problem.shape[1] + dims.n_pol_transform = problem.transform.n_pol + dims.n_nodes = problem.transform.n_nodes + # solprop SEP 47 section 6.5: the backend's execution choice, lowered by + # `native_node_chunk`. + dims.node_chunk = node_chunk # solprop SEP 6's number-conserving sector is formed on analytic signals, of the - # filtered fields and, for the polynomial, of the bare one; a Raman drive beside it - # is the cycle-averaged one, which reads the analytic signal too. - # The general modal route synthesises its own analytic signal on the real family. - family = ( - "envelope" + # filtered fields and, for the polynomial, of the bare one; a Raman drive beside + # it is the cycle-averaged one, which reads the analytic signal too. The general + # modal route synthesises its own analytic signal on the real family. + dims.family = ( + _core.FieldFamily.envelope if problem.grid_kind == "envelope" - else "field_analytic" + else _core.FieldFamily.field_analytic if (kerr is not None and kerr.analytic_signal) or ( dispersive is not None and dispersive.sector == "conserving" and problem.transform.modal_tables is None ) - else "field_real" + else _core.FieldFamily.field_real ) - absorber = problem.responses.temporal_absorber - absorber_fine = problem.responses.temporal_absorber_fine - spatial_absorber = radial_absorber_matrix( - problem.transform, problem.grids.pad_scale + + grids = problem.grids + carrier = grids.carrier_2w0 + grid = native.grid + grid.crop_index = np.ascontiguousarray(grids.crop_index, dtype=np.uint64) + grid.active_mask = np.ascontiguousarray(grids.active_mask) + grid.norm_mask = np.ascontiguousarray(grids.norm_mask) + grid.omega_window = np.ascontiguousarray(grids.omega_window) + grid.omega = np.ascontiguousarray(grids.omega) + grid.time_window = np.ascontiguousarray(grids.t_window_fine) + if problem.grid_kind == "field" and problem.responses.plasma_channels: + grid.time_window_derivative = np.ascontiguousarray( + grids.t_window_fine_derivative + ) + if geometry is None: + grid.power_weight = np.ascontiguousarray(power_weight_at(problem, 0.0)) + grid.carrier = np.ascontiguousarray( + np.ones(grids.n_to, dtype=np.complex128) if carrier is None else carrier ) - temporal_in_stage = ( - absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" + grid.dt_fine = grids.dt_fine + grid.dt = grids.dt + + _lower_linear(native, problem, plan) + coarse_matrix = _lower_spatial(native, problem, request) + _lower_stepper(native, problem, stepper, device=plan.device) + + native.absorber.in_stage = temporal_in_stage or spatial_absorption + native.absorber.split_step = ( + absorber is not None and problem.responses.absorber_scheme == "split_step" ) - carrier = problem.grids.carrier_2w0 - n_omega = problem.shape[-1] - phi_terms = fixed.phi - prefactor_terms = fixed.prefactor - spatial = fixed.separable - if spatial is not None: - phi = np.empty((1, 0, n_omega), dtype=np.complex128) - prefactor = np.empty((1, 0, n_omega), dtype=np.complex128) + if temporal_in_stage: + native.absorber.fine = np.ascontiguousarray(absorber_fine, dtype=np.float64) + elif spatial_absorption: + native.absorber.fine = np.zeros(grids.n_to) + if absorber is not None: + native.absorber.coarse = np.ascontiguousarray(absorber, dtype=np.float64) + + transform = native.transform + transform.synthesis_matrix = np.ascontiguousarray(problem.transform.synthesis) + transform.projection_matrix = np.ascontiguousarray(problem.transform.projection) + transform.projection_coarse_matrix = np.ascontiguousarray(coarse_matrix) + transform.synthesis = 0.0 if free else float(problem.transform.synthesis[0, 0]) + transform.projection = 0.0 if free else float(problem.transform.projection[0, 0]) + + if kerr is not None: + native.kerr.nonlinear = True + native.kerr.coefficient = kerr.coefficient + native.kerr.self_factor = kerr.self_factor + native.kerr.conjugate_factor = kerr.conjugate_factor + native.kerr.harmonic_factor = kerr.harmonic_factor + native.kerr.take_real = kerr.take_real + + _lower_plasma(native, problem, request) + _lower_raman(native, problem) + _lower_general(native, problem, general_chunk) + _lower_flow(native, plan.flow) + _lower_dispersive(native, problem) + _lower_flux(native, problem, request.observation.stats) + return native + + +def _lower_linear( + native: _core.Problem, problem: PropagationProblem, plan: _LowerPlan +) -> None: + """Fill the fixed terms: one empty term on a separable route, a z table's slots.""" + del problem + tables = plan.tables + linear = native.linear + linear.phi_table = tables is not None and tables.phi + linear.prefactor_table = tables is not None and tables.prefactor + if plan.terms.separable is not None: + linear.n_phi_terms = 1 + linear.n_prefactor_terms = 1 + return + # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): + # their count crosses in their place. + if tables is not None and tables.phi: + linear.n_phi_terms = tables.n_phi + elif plan.terms.phi is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") else: - # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): - # an empty array whose leading extent is their count crosses in their place. - if tables is not None and tables.phi: - phi = np.empty((tables.n_phi, 0), dtype=np.complex128) - elif phi_terms is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - phi = np.ascontiguousarray( - phi_terms.reshape(phi_terms.shape[0], -1, n_omega) - ) - if tables is not None and tables.prefactor: - prefactor = np.empty((tables.slots, 0), dtype=np.complex128) - elif prefactor_terms is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - prefactor = np.ascontiguousarray( - prefactor_terms.reshape(prefactor_terms.shape[0], -1, n_omega) - ) - channels = problem.responses.population_channels - raman_terms = problem.responses.raman_terms - plasma = channels[0] if channels else None - raman = raman_terms[0] if raman_terms else None - rate = None if plasma is None else plasma.rate + linear.n_phi_terms = plan.terms.phi.shape[0] + linear.phi_terms = np.ascontiguousarray(plan.terms.phi) + if tables is not None and tables.prefactor: + linear.n_prefactor_terms = tables.slots + elif plan.terms.prefactor is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") + else: + linear.n_prefactor_terms = plan.terms.prefactor.shape[0] + linear.prefactor_terms = np.ascontiguousarray(plan.terms.prefactor) + + +def _lower_spatial( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> np.ndarray: + """Fill the separable operator and a radial run's rows; return the coarse matrix. + + The quadrature alarm's embedded Kronrod projection is the transform's own unless + it is the plain projection or the route has one node, and the hybrid planar kind + packs none: the reference scores no alarm on it. + """ + transform = problem.transform coarse = ( np.empty(0, dtype=np.float64) - if problem.transform.n_nodes == 1 - or np.array_equal( - problem.transform.projection_coarse, problem.transform.projection - ) - else problem.transform.projection_coarse + if transform.n_nodes == 1 + or np.array_equal(transform.projection_coarse, transform.projection) + else transform.projection_coarse ) - geometry = problem.transform.geometry - # Free space and schema 15's hybrid planar kind share the separable operator and - # the transverse FFT engine; the planar kind adds its slab rule in x. - free = geometry is not None and geometry.kind in ("free", "slab") - spatial_absorption = bool(spatial_absorber.size) or ( - free and geometry is not None and bool(np.any(geometry.absorber_rate)) + geometry = transform.geometry + spatial = native.spatial + spatial.absorber_matrix = radial_absorber_matrix(transform, problem.grids.pad_scale) + if geometry is not None: + spatial.spectral_weights = geometry.spectral_weights + spatial.on_axis = geometry.on_axis + separable = problem.linear.separable + if geometry is None or geometry.kind not in ("free", "slab") or separable is None: + return coarse + spatial.n_x, spatial.n_y = geometry.shape + spatial.k_squared = separable.k_squared + spatial.k_squared_low = separable.k_squared_low + spatial.kperp_sq = separable.kperp_sq + spatial.kperp_sq_low = separable.kperp_sq_low + spatial.frame = separable.frame + spatial.spectral_rate = separable.spectral_rate + assert separable.user_rate is not None + assert separable.kperp_rate is not None + spatial.user_rate = separable.user_rate + spatial.numerator = separable.prefactor_numerator + spatial.clamped = separable.prefactor_clamped + spatial.evanescent_rate = separable.evanescent_rate + spatial.absorber_rate = geometry.absorber_rate + # A transverse collar's rate per column (solprop kernel schema 31). + spatial.kperp_rate = separable.kperp_rate + if geometry.kind == "free": + # solprop SEP 23: the stored axes and the slots of the problem's own + # far-field request, which size the native band partials. + k_x, k_y = geometry.spectral_axes + slots, _ = far_field_partials(problem, request.observation.stats) + spatial.k_x = np.ascontiguousarray(k_x, dtype=np.float64) + spatial.k_y = np.ascontiguousarray(k_y, dtype=np.float64) + spatial.far_field_slots = slots + return coarse + # Schema 15's `SeparableOperator.kappa_sq` is (n_channels, n_omega), column `c` + # belonging to channel `c % n_channels`; the geometry's collar rate depends on y + # alone (solprop checks it), so it is packed per column `p + n_p * iy`, the index + # of the temporal row it acts on. + assert separable.kappa_sq is not None + n_x, n_y = geometry.shape + n_p = separable.n_channels + spatial.n_p = n_p + spatial.kappa_sq = np.ascontiguousarray(separable.kappa_sq) + spatial.absorber_rate = np.ascontiguousarray( + np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) ) - coarse_matrix = coarse - spatial_data: dict[str, object] = {} - if free and geometry is not None and spatial is not None: - spatial_data = { - "n_x": geometry.shape[0], - "n_y": geometry.shape[1], - "spatial_k_squared": spatial.k_squared, - "spatial_k_squared_low": spatial.k_squared_low, - "spatial_kperp_sq": spatial.kperp_sq, - "spatial_kperp_sq_low": spatial.kperp_sq_low, - "spatial_frame": spatial.frame, - "spatial_spectral_rate": spatial.spectral_rate, - "spatial_user_rate": spatial.user_rate, - "spatial_numerator": spatial.prefactor_numerator, - "spatial_clamped": spatial.prefactor_clamped, - "spatial_evanescent_rate": spatial.evanescent_rate, - "spatial_absorber_rate": geometry.absorber_rate, - # A transverse collar's rate per column (solprop kernel schema 31). - "spatial_kperp_rate": spatial.kperp_rate, - } - if geometry.kind == "free": - # solprop SEP 23: the stored axes and the slots of the problem's own - # far-field request, which size the native band partials. - k_x, k_y = geometry.spectral_axes - slots, _ = far_field_partials(problem, request.observation.stats) - spatial_data |= { - "spatial_k_x": np.ascontiguousarray(k_x, dtype=np.float64), - "spatial_k_y": np.ascontiguousarray(k_y, dtype=np.float64), - "spatial_far_field_slots": slots, - } - if geometry.kind == "slab": - spatial_data |= _slab_payload(problem) - # The reference scores no quadrature alarm on this kind, so no embedded - # Kronrod projection is packed. - coarse_matrix = np.empty(0, dtype=np.float64) - # The accepted step's last stage leaves the plasma statistics behind only when - # that stage's input is the committed field: a first-same-as-last - # tableau, no split-step absorber (applied to the field after the last stage) - # and no coupled flow; and not under the modified noise model, where that input - # carries the noise draw while solprop defines the statistics on the committed - # field its reference scans. Otherwise they are scanned afresh. - stash_plasma = ( - plasma is not None + return np.empty(0, dtype=np.float64) + + +def _lower_stepper( + native: _core.Problem, + problem: PropagationProblem, + stepper: StepperSettings, + *, + device: bool, +) -> None: + """Fill the tableau and the low-storage march's rows (solprop kernel schema 32).""" + tableau = stepper.tableau + native_stepper = native.stepper + native_stepper.n_stages = tableau.n_stages + native_stepper.tableau_matrix = np.ascontiguousarray(tableau.matrix) + native_stepper.tableau_nodes = np.ascontiguousarray(tableau.nodes) + native_stepper.solution_weights = np.ascontiguousarray(tableau.solution_weights) + native_stepper.error_weights = np.ascontiguousarray(tableau.error_weights) + native_stepper.first_same_as_last = tableau.first_same_as_last + native_stepper.weighted_controller = stepper.controller.weights is not None + native_stepper.modified_noise = problem.initial.noise_model == "modified" + flows = _low_storage_flows(problem, stepper, device=device) + if flows is None: + return + native_stepper.low_storage = True + native_stepper.low_storage_stage_register = low_storage_stage_register( + problem, "native" + ) + native_stepper.low_storage_flows = flows + native_stepper.low_storage_tiled_input = device + + +def _lower_plasma( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> None: + """Fill the plasma columns: the first channel's rate, every channel's rows.""" + channels = problem.responses.population_channels + channel = channels[0] if channels else None + plasma = native.plasma + # solprop SEP 18: the running-integral rule, one per problem. + plasma.band = problem.responses.plasma_integral.policy == "band" + plasma.taps = problem.responses.plasma_integral.taps + if channel is not None: + rate = channel.rate + plasma.enabled = True + # The private gated benchmark (solprop's tests/gated_source.py), read where + # the backend is built, as the reference reads it. + plasma.gated = _BENCHMARK_GATED.get() + plasma.charge_mass_ratio = channel.charge_mass_ratio + plasma.ionisation_potential = channel.ionisation_potential + plasma.log_field_min = rate.log_field_min + plasma.d_log_field = rate.d_log_field + plasma.field_min = rate.field_min + plasma.field_max = rate.field_max + plasma.rate_max = rate.rate_max + plasma.rate_intervals = rate.coefficients.shape[1] + plasma.rate_coefficients = np.ascontiguousarray(rate.coefficients) + plasma.stash = _stash_plasma(problem, request) + _lower_channels(native, problem) + _lower_envelope(native, problem) + + +def _stash_plasma(problem: PropagationProblem, request: RunRequest) -> bool: + """Whether the accepted step's last stage leaves the plasma statistics behind. + + Only when that stage's input is the committed field: a first-same-as-last tableau, + no split-step absorber (applied to the field after the last stage) and no coupled + flow; and not under the modified noise model, where that input carries the noise + draw while solprop defines the statistics on the committed field its reference + scans. Otherwise they are scanned afresh. + """ + absorber = problem.responses.temporal_absorber + return ( + bool(problem.responses.population_channels) and problem.responses.envelope_plasma is None and problem.initial.noise_model != "modified" - and stepper.tableau.first_same_as_last + and request.stepper.tableau.first_same_as_last and not ( absorber is not None and problem.responses.absorber_scheme == "split_step" ) @@ -3438,133 +3561,35 @@ def _native_problem( & set(request.observation.stats.periodic) ) ) - return { - # solprop SEP 46: how the workspace's CPU operations thread, the process's - # record unless the backend was given one. - "cpu_execution": _package.EXECUTION.native(), - **spatial_data, - **_channel_payload(problem), - **_envelope_payload(problem), - **_general_payload(problem, general_chunk), - **_flow_payload(flow), - **_dispersive_payload(problem), - **_flux_payload(problem, request.observation.stats), - "n_omega": n_omega, - "n_t": problem.grids.n_t, - "n_to": problem.grids.n_to, - "n_stages": stepper.tableau.n_stages, - "n_modes": problem.shape[0], - "n_pol_state": problem.shape[1], - "n_pol_transform": problem.transform.n_pol, - "n_nodes": problem.transform.n_nodes, - # solprop SEP 47 section 6.5: the backend's execution choice, lowered by - # `native_node_chunk`. - "node_chunk": node_chunk, - "family": family, - "nonlinear": kerr is not None, - "modified_noise": problem.initial.noise_model == "modified", - "in_stage_absorber": temporal_in_stage or spatial_absorption, - "split_step_absorber": absorber is not None - and problem.responses.absorber_scheme == "split_step", - "first_same_as_last": stepper.tableau.first_same_as_last, - "crop_index": np.ascontiguousarray(problem.grids.crop_index, dtype=np.uint64), - "active_mask": np.ascontiguousarray(problem.grids.active_mask), - "norm_mask": np.ascontiguousarray(problem.grids.norm_mask), - "omega_window": np.ascontiguousarray(problem.grids.omega_window), - "omega": np.ascontiguousarray(problem.grids.omega), - "time_window": np.ascontiguousarray(problem.grids.t_window_fine), - "time_window_derivative": np.ascontiguousarray( - problem.grids.t_window_fine_derivative - if problem.grid_kind == "field" and problem.responses.plasma_channels - else np.empty(0) - ), - "absorber_fine": np.ascontiguousarray( - absorber_fine - if temporal_in_stage - else np.zeros(problem.grids.n_to) - if spatial_absorption - else np.empty(0), - dtype=np.float64, - ), - "spatial_absorber_matrix": spatial_absorber, - "spatial_spectral_weights": np.empty(0) - if geometry is None - else geometry.spectral_weights, - "spatial_on_axis": np.empty(0, dtype=np.complex128) - if geometry is None - else geometry.on_axis, - "absorber_coarse": np.ascontiguousarray( - np.empty(0) if absorber is None else absorber, dtype=np.float64 - ), - "carrier": np.ascontiguousarray( - np.ones(problem.grids.n_to, dtype=np.complex128) - if carrier is None - else carrier - ), - "synthesis_matrix": np.ascontiguousarray(problem.transform.synthesis), - "projection_matrix": np.ascontiguousarray(problem.transform.projection), - "projection_coarse_matrix": np.ascontiguousarray(coarse_matrix), - "phi_terms": phi, - "prefactor_terms": prefactor, - "phi_table": tables is not None and tables.phi, - "prefactor_table": tables is not None and tables.prefactor, - "power_weight": np.ascontiguousarray( - np.empty(0) if geometry is not None else power_weight_at(problem, 0.0) - ), - "tableau_matrix": np.ascontiguousarray(stepper.tableau.matrix), - "tableau_nodes": np.ascontiguousarray(stepper.tableau.nodes), - "solution_weights": np.ascontiguousarray(stepper.tableau.solution_weights), - "error_weights": np.ascontiguousarray(stepper.tableau.error_weights), - "synthesis": 0.0 if free else float(problem.transform.synthesis[0, 0]), - "projection": 0.0 if free else float(problem.transform.projection[0, 0]), - "kerr_coefficient": 0.0 if kerr is None else kerr.coefficient, - "self_factor": 0.0 if kerr is None else kerr.self_factor, - "conjugate_factor": 0.0 if kerr is None else kerr.conjugate_factor, - "harmonic_factor": 0.0 if kerr is None else kerr.harmonic_factor, - "take_real": kerr is not None and kerr.take_real, - "plasma": plasma is not None, - # The private gated benchmark (solprop's tests/gated_source.py), read where - # the backend is built, as the reference reads it. - "plasma_gated": plasma is not None and _BENCHMARK_GATED.get(), - "raman": raman is not None, - "raman_thg": raman is not None - and bool(raman.thg) - and problem.grid_kind == "field", - "raman_tensor_capable": bool(raman_terms) - and all(term.tensor_capable for term in raman_terms), - "raman_tensor_weights": np.ascontiguousarray( - np.concatenate([term.tensor_weights for term in raman_terms], axis=0) - if raman_terms - else np.empty((0, 2), dtype=np.float64) - ), - "stash_plasma": stash_plasma, - "weighted_controller": stepper.controller.weights is not None, - "dt_fine": problem.grids.dt_fine, - "dt": problem.grids.dt, - "raman_fraction": 0.0 if raman is None else float(raman.fraction), - "raman_epsilon0": EPS0 if raman is not None else 0.0, - "raman_undamped": np.ascontiguousarray( - np.empty((0, 2 * problem.grids.n_to), dtype=np.float64) - if raman is None - else np.concatenate([term.undamped for term in raman_terms], axis=0) - ), - "charge_mass_ratio": 0.0 if plasma is None else plasma.charge_mass_ratio, - "ionisation_potential": 0.0 if plasma is None else plasma.ionisation_potential, - "log_field_min": 0.0 if rate is None else rate.log_field_min, - "d_log_field": 0.0 if rate is None else rate.d_log_field, - "field_min": 0.0 if rate is None else rate.field_min, - "field_max": 0.0 if rate is None else rate.field_max, - "rate_max": 0.0 if rate is None else rate.rate_max, - "rate_coefficients": np.ascontiguousarray( - np.empty((4, 0), dtype=np.float64) if rate is None else rate.coefficients - ), - # solprop SEP 18: the running-integral rule, one per problem. - "plasma_band": problem.responses.plasma_integral.policy == "band", - "plasma_taps": problem.responses.plasma_integral.taps, - } -def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: +def _lower_raman(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the Raman terms, row-major (n_terms, 2 n_to) and (n_terms, 2).""" + terms = problem.responses.raman_terms + raman = native.raman + raman.n_components = len(terms) + raman.row_component = np.asarray( + [component for component, term in enumerate(terms) for _ in term.undamped], + dtype=np.float64, + ) + raman.component_fraction = np.asarray([term.fraction for term in terms]) + if not terms: + return + first = terms[0] + undamped = np.concatenate([term.undamped for term in terms], axis=0) + raman.enabled = True + raman.thg = bool(first.thg) and problem.grid_kind == "field" + raman.tensor_capable = all(term.tensor_capable for term in terms) + raman.n_terms = undamped.shape[0] + raman.fraction = float(first.fraction) + raman.epsilon0 = EPS0 + raman.undamped = np.ascontiguousarray(undamped) + raman.tensor_weights = np.ascontiguousarray( + np.concatenate([term.tensor_weights for term in terms], axis=0) + ) + + +def _lower_dispersive(native: _core.Problem, problem: PropagationProblem) -> None: """Lay out solprop SEP 6's dispersive response for the native engine. The structure the native response takes on every route: the channels grouped as the @@ -3583,7 +3608,7 @@ def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: """ leaf = problem.responses.dispersive_kerr if leaf is None: - return {"dispersive": False} + return tables = problem.transform.modal_tables general = tables is not None kind = "field" if general else problem.grid_kind @@ -3655,70 +3680,62 @@ def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: else: quartics = np.empty(0, dtype=np.float64) offsets = np.cumsum([0, *(len(group) for group in groups)]) - route: dict[str, object] = {} + lowered = native.dispersive if tables is not None: - route = { - "dispersive_group_keys": np.asarray(keys, dtype=np.uint64), - "dispersive_node_region": np.ascontiguousarray( - tables.region_index, dtype=np.uint64 - ), - } - return { - **route, - "dispersive": True, - "dispersive_symmetric": symmetric, - "dispersive_form": {"real": 0, "conserving": 1, "harmonic": 2}[form], - "dispersive_inputs": np.ascontiguousarray(stacked), - "dispersive_term_multipliers": np.ascontiguousarray( - np.array(multipliers, dtype=np.float64) - ), - "dispersive_term_rows": np.asarray(rows, dtype=np.uint64), - "dispersive_term_channels": np.asarray(channels, dtype=np.uint64), - "dispersive_coefficients": np.asarray(leaf.coefficients, dtype=np.float64), - "dispersive_pair_constants": np.ascontiguousarray(constants), - "dispersive_group_offsets": np.asarray(offsets, dtype=np.uint64), - "dispersive_group_members": np.asarray( - [channel for group in groups for channel in group], dtype=np.uint64 - ), - "dispersive_quartics": quartics, - "dispersive_input_scale": np.ascontiguousarray(input_scale), - "dispersive_output_scale": np.ascontiguousarray(scale), - } + lowered.group_keys = np.asarray(keys, dtype=np.uint64) + lowered.node_region = np.ascontiguousarray(tables.region_index, dtype=np.uint64) + lowered.enabled = True + lowered.symmetric = symmetric + lowered.form = {"real": 0, "conserving": 1, "harmonic": 2}[form] + lowered.inputs = np.ascontiguousarray(stacked) + lowered.term_multipliers = np.ascontiguousarray( + np.array(multipliers, dtype=np.float64) + ) + lowered.term_rows = np.asarray(rows, dtype=np.uint64) + lowered.term_channels = np.asarray(channels, dtype=np.uint64) + lowered.coefficients = np.asarray(leaf.coefficients, dtype=np.float64) + lowered.pair_constants = np.ascontiguousarray(constants) + lowered.group_offsets = np.asarray(offsets, dtype=np.uint64) + lowered.group_members = np.asarray( + [channel for group in groups for channel in group], dtype=np.uint64 + ) + lowered.quartics = quartics + lowered.input_scale = np.ascontiguousarray(input_scale) + lowered.output_scale = np.ascontiguousarray(scale) -def _flux_payload(problem: PropagationProblem, stats: StatsSpec) -> dict[str, object]: - """Pack the plane measure of solprop SEP 34 (kernel schema 35). +def _lower_flux( + native: _core.Problem, problem: PropagationProblem, stats: StatsSpec +) -> None: + """Fill the plane measure of solprop SEP 34 (kernel schema 35). A separable run (Cartesian, line, planar) is measured by its own operator's root, which the native state already holds: nothing more is packed, and a plan that is not that operator is refused. A radial run packs the entrance's material and transverse squares and whether a material z table follows - (:func:`_upload_flux_table`); the admission record is not read. ``flux_closure`` + (:func:`_upload_flux_table`); the admission record is not read. ``closure`` sizes the magnetic closure's buffers for the run's own local statistics, `stats`. """ flux = flux_of(problem.route) if flux is None: - return {} - closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) + return + native.flux.closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) if problem.linear.separable is not None: if flux.base is not problem.linear.separable or flux.material is not None: raise BackendError( "solprop-native measures a separable run by its own operator's root, " "and this plane measure is another" ) - return {"flux_closure": closure} + return base = flux.base - return { - "flux_k_squared": np.ascontiguousarray(base.k_squared, dtype=np.complex128), - "flux_k_squared_low": np.ascontiguousarray( - base.k_squared_low, dtype=np.complex128 - ), - "flux_kperp_sq": np.ascontiguousarray(base.kperp_sq, dtype=np.float64), - "flux_kperp_sq_low": np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64), - "flux_table": flux.material is not None, - "flux_closure": closure, - } + native.flux.k_squared = np.ascontiguousarray(base.k_squared, dtype=np.complex128) + native.flux.k_squared_low = np.ascontiguousarray( + base.k_squared_low, dtype=np.complex128 + ) + native.flux.kperp_sq = np.ascontiguousarray(base.kperp_sq, dtype=np.float64) + native.flux.kperp_sq_low = np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64) + native.flux.table = flux.material is not None def _upload_flux_table(native: _core.State, problem: PropagationProblem) -> None: @@ -3741,111 +3758,78 @@ def _upload_flux_table(native: _core.State, problem: PropagationProblem) -> None ) -def _slab_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack the hybrid planar kind's channels, wall terms and per-column collar rate. - - Schema 15's ``SeparableOperator.kappa_sq`` is ``(n_channels, n_omega)``, column - ``c`` belonging to channel ``c % n_channels``; the geometry's collar rate depends on - y alone (solprop checks it), so it is packed per column ``p + n_p * iy``, the index - of the temporal row it acts on. - """ - geometry = problem.transform.geometry - separable = problem.linear.separable - assert geometry is not None - assert separable is not None - assert separable.kappa_sq is not None - n_x, n_y = geometry.shape - n_p = separable.n_channels - return { - "n_p": n_p, - "spatial_kappa_sq": np.ascontiguousarray(separable.kappa_sq), - "spatial_absorber_rate": np.ascontiguousarray( - np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) - ), - } - - -def _general_payload( - problem: PropagationProblem, chunk: int | None -) -> dict[str, object]: - """Pack the general modal route's maps, derived as the reference derives them. +def _lower_general( + native: _core.Problem, problem: PropagationProblem, chunk: int | None +) -> None: + """Fill the general modal route's maps, derived as the reference derives them. The per-node tensor, the form and the delayed maps come from solprop's own derivations (``solprop.translate.general``), so both backends execute the same arrays; the chunk is the route's own at the same budget (:func:`solprop.backends.node_chunk.general_planned_chunk`), given as `chunk`. - Nothing is packed for a problem without ``transform.modal_tables``. + Nothing is filled for a problem without ``transform.modal_tables``. """ tables = problem.transform.modal_tables if tables is None: - return {"general_modal": False} + return assert tables.synthesis is not None assert tables.projection is not None - regional = problem.responses.regional_responses - components = int(tables.electric.shape[2]) - nodes = int(tables.region_index.size) assert chunk is not None + regional = problem.responses.regional_responses owner, delayed_tensor, delayed_spectrum = node_delayed_maps(tables, regional) terms = 0 if delayed_tensor is None else int(delayed_tensor.shape[1]) - intervals, n_deg = (int(one) for one in tables.synthesis.shape[:2]) - return { - "general_modal": True, - "general_real_form": node_kerr_form(regional) == "real", - # A moving guide's maps are every Chebyshev coefficient along z, flattened - # over (n_intervals, n_deg) in the order of the stage bundle's - # transform_weights; a constant guide's are the one coefficient. - "general_map_intervals": intervals, - "general_map_n_deg": n_deg, - "general_components": components, - "general_nodes": nodes, - "general_node_chunk": chunk, - "general_responses": 0 if delayed_tensor is None else delayed_tensor.shape[0], - "general_terms": terms, - "general_pad_scale": float(problem.grids.pad_scale), - "general_epsilon0": EPS0, - "general_synthesis": np.ascontiguousarray(tables.synthesis), - "general_projection": np.ascontiguousarray(tables.projection), - "general_tensor": np.ascontiguousarray(node_kerr_tensor(tables, regional)), - "general_frequency_factor": np.ascontiguousarray( - tables.source_frequency_factor, dtype=np.complex128 - ), - "general_delayed_tensor": np.ascontiguousarray( - np.empty(0) if delayed_tensor is None else delayed_tensor - ), - "general_delayed_spectrum": np.ascontiguousarray( - np.empty(0, dtype=np.complex128) - if delayed_spectrum is None - else delayed_spectrum - ), - "general_node_response": np.ascontiguousarray( - np.empty(0, dtype=np.uint64) if not terms else owner.astype(np.int64) + 1, - dtype=np.uint64, - ), - } + general = native.general + general.enabled = True + general.real_form = node_kerr_form(regional) == "real" + # A moving guide's maps are every Chebyshev coefficient along z, flattened over + # (n_intervals, n_deg) in the order of the stage bundle's transform_weights; a + # constant guide's are the one coefficient. + general.map_intervals, general.map_n_deg = ( + int(one) for one in tables.synthesis.shape[:2] + ) + general.components = int(tables.electric.shape[2]) + general.nodes = int(tables.region_index.size) + general.node_chunk = chunk + general.responses = 0 if delayed_tensor is None else delayed_tensor.shape[0] + general.terms = terms + general.pad_scale = float(problem.grids.pad_scale) + general.epsilon0 = EPS0 + general.synthesis = np.ascontiguousarray(tables.synthesis) + general.projection = np.ascontiguousarray(tables.projection) + general.tensor = np.ascontiguousarray(node_kerr_tensor(tables, regional)) + general.frequency_factor = np.ascontiguousarray( + tables.source_frequency_factor, dtype=np.complex128 + ) + if delayed_tensor is not None: + general.delayed_tensor = np.ascontiguousarray(delayed_tensor) + if delayed_spectrum is not None: + general.delayed_spectrum = np.ascontiguousarray(delayed_spectrum) + if terms: + general.node_response = np.ascontiguousarray( + owner.astype(np.int64) + 1, dtype=np.uint64 + ) -def _envelope_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack the accepted cycle rules and already masked spectral factors.""" +def _lower_envelope(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the accepted cycle rules, (n_channels, 4, 512), and masked factors.""" assembly = problem.responses.envelope_plasma if assembly is None: - return {"envelope_plasma": False} + return nodes = np.zeros((len(assembly.channels), 4, 512)) weights = np.zeros_like(nodes) for channel_index, channel in enumerate(assembly.channels): for level, rule in enumerate(channel.cycle.rules): nodes[channel_index, level, : len(rule.nodes)] = rule.nodes weights[channel_index, level, : len(rule.weights)] = rule.weights - return { - "envelope_plasma": True, - "envelope_rule_nodes": nodes, - "envelope_rule_weights": weights, - "envelope_drude_factor": assembly.factors.drude, - "envelope_loss_factor": assembly.factors.loss, - } + native.plasma.envelope = True + native.plasma.envelope_rule_nodes = nodes + native.plasma.envelope_rule_weights = weights + native.plasma.envelope_drude_factor = assembly.factors.drude + native.plasma.envelope_loss_factor = assembly.factors.loss -def _channel_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack independent rate tables without resampling their coefficients.""" +def _lower_channels(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill independent rate tables without resampling their coefficients.""" parameters: list[list[float]] = [] coefficients: list[np.ndarray] = [] offset = 0 @@ -3866,17 +3850,45 @@ def _channel_payload(problem: PropagationProblem) -> dict[str, object]: ) coefficients.append(rate.coefficients.ravel()) offset += rate.coefficients.size - terms = problem.responses.raman_terms - return { - "n_channels": len(parameters), - "channel_parameters": np.asarray(parameters, dtype=np.float64).reshape(-1, 9), - "channel_coefficients": np.concatenate(coefficients) - if coefficients - else np.empty(0), - "n_raman_components": len(terms), - "raman_row_component": np.asarray( - [component for component, term in enumerate(terms) for _ in term.undamped], - dtype=np.float64, - ), - "raman_component_fraction": np.asarray([term.fraction for term in terms]), - } + native.plasma.n_channels = len(parameters) + native.plasma.channel_parameters = np.asarray(parameters, dtype=np.float64).reshape( + -1, 9 + ) + if coefficients: + native.plasma.channel_coefficients = np.concatenate(coefficients) + + +def _lower_flow(native: _core.Problem, flow: _HostFlow | None) -> None: + """Fill a coupled flow's bins and residual data; its actions follow per step.""" + if flow is None: + return + slot = np.zeros(flow.active.size, dtype=np.uint64) + slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) + residual = flow.residual_terms + # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major + # (section, bin) order and carry their Schur factors; the rest take the + # eigenbasis, whose `W` and `Y` are zero where it is not used. + fallback = ~flow.route + route = np.zeros(flow.route.shape, dtype=np.uint64) + route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) + lowered = native.flow + lowered.enabled = True + lowered.active = int(np.count_nonzero(flow.active)) + lowered.sections = int(flow.frozen.shape[0]) + lowered.terms = 0 if residual is None else int(residual.shape[0]) + lowered.slot = slot + if residual is not None: + # The residual's data are zero on the bins the error norm does not score, so + # the engine's residual there is an exact zero and the frozen flow alone + # carries them, as in solprop's reference. Every term is finite, so each + # product with a zero is a signed zero and the running sums stay +0: the + # source is what adding nothing leaves, but for the sign of a zero. + scored = flow.scored[None, :, None, None] + lowered.frozen = np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)) + lowered.residual_terms = np.ascontiguousarray(np.where(scored, residual, 0.0)) + lowered.left = flow.left + lowered.right = flow.right + lowered.eigenvalues = flow.eigenvalues + lowered.route = route + lowered.fallback_vectors = np.ascontiguousarray(flow.vectors[fallback]) + lowered.fallback_triangular = np.ascontiguousarray(flow.triangular[fallback]) diff --git a/native/src/solprop_native/session.py b/native/src/solprop_native/session.py index 35cfa29a9..2055fdbb1 100644 --- a/native/src/solprop_native/session.py +++ b/native/src/solprop_native/session.py @@ -152,7 +152,7 @@ _NO_SCHEDULE, # pyright: ignore[reportPrivateUsage] NativeBackend, _aperture_payload, # pyright: ignore[reportPrivateUsage] - _low_storage_payload, # pyright: ignore[reportPrivateUsage] + _low_storage_flows, # pyright: ignore[reportPrivateUsage] _on_device, # pyright: ignore[reportPrivateUsage] _raise_native, # pyright: ignore[reportPrivateUsage] ) @@ -604,11 +604,12 @@ def _prepare( kind = "low_storage" # The flows `_ResidentLowStorage` holds, and the operator's origin, from # which a constant operator's table is built. - payload = _low_storage_payload( + flows = _low_storage_flows( self._problem, self._stepper, device=_on_device() ) + assert flows is not None # the march is the low-storage one backend._native_state.low_storage_configure( # pyright: ignore[reportPrivateUsage] - int(cast("int", payload["low_storage_flows"])), + flows, driver._terms.separable is not None, float(self._problem.linear.z_span[0]), ) diff --git a/native/src/transverse.cpp b/native/src/transverse.cpp index ddb347542..18617139e 100644 --- a/native/src/transverse.cpp +++ b/native/src/transverse.cpp @@ -210,11 +210,11 @@ void Edge::resume(const Resumption& record) { return; resumed_ = true; launch_ = launch; - baseline_ = *record.edge_baseline; + baseline_ = record.edge_baseline.value(); if (fits(record.edge_max)) - peak_ = *record.edge_max; + peak_ = record.edge_max.value(); if (fits(record.edge_max_raw)) - peak_raw_ = *record.edge_max_raw; + peak_raw_ = record.edge_max_raw.value(); allowance_ = record.noise_allowance.value_or(0.0); ramp_raw_ = record.raw_ramp_removed.value_or(0.0); ramp_noise_ = record.noise_ramp_removed.value_or(0.0); @@ -340,8 +340,8 @@ Notice Edge::run_launch() { case Stage::launch_clip: while (face_ < n_faces_) { const std::size_t face = face_++; - baseline_[face] = (*held_.faces)[face] / launch_; - const double clipped = (*held_.clip)[face] / launch_; + baseline_[face] = held_.faces.value()[face] / launch_; + const double clipped = held_.clip.value()[face] / launch_; if (clipped > config_.warn) { Notice notice = notice_of(Notice::Kind::clip, clipped); notice.face = face; @@ -463,7 +463,7 @@ Notice Edge::judge_faces(const Reading& reading, const double z) { // `max` of `(judged, name)` tuples, ties broken on the name. std::array judged{}; for (std::size_t face = 0; face < n_faces_; ++face) { - const double share = (*reading.faces)[face] / launch_; + const double share = reading.faces.value()[face] / launch_; peak_raw_[face] = py_max(peak_raw_[face], share); judged[face] = share - baseline_[face] - allowance_; peak_[face] = py_max(peak_[face], judged[face]); diff --git a/native/src/watch.cpp b/native/src/watch.cpp index 79e6d4bb7..785bcd8d1 100644 --- a/native/src/watch.cpp +++ b/native/src/watch.cpp @@ -4,6 +4,8 @@ #include "solprop_native/watch.hpp" +#include "solprop_native/refuse.hpp" + #include #include #include @@ -43,8 +45,8 @@ double py_min(const double a, const double b) { return b < a ? b : a; } /** NumPy's `maximum(x, 0.0)`: the larger, a NaN kept. */ double clip_low(const double x) { return std::isnan(x) || x >= 0.0 ? x : 0.0; } -[[noreturn]] void refuse(const std::string& field) { - throw std::invalid_argument("the boundary watch's policy has an invalid " + field); +[[noreturn]] void refuse_policy(const char* field) { + refuse("the boundary watch's policy has an invalid ", field); } /** `_settle`: the cadence and the quiet flag from a check's shares. */ @@ -111,15 +113,15 @@ Verdict judge_shares(const Policy& policy, State& state, const double* shares, void validate(const Policy& policy) { if (policy.cadence < 1) - refuse("cadence"); + refuse_policy("cadence"); if (!(policy.interval > 0.0)) - refuse("interval"); + refuse_policy("interval"); if (!(policy.warn > 0.0) || !(policy.stop >= policy.warn)) - refuse("warning or stop threshold"); + refuse_policy("warning or stop threshold"); if (!(policy.activity >= 0.0) || !(policy.launch_floor >= 0.0)) - refuse("activity threshold or launch floor"); + refuse_policy("activity threshold or launch floor"); if (policy.crossing && policy.bucket_limits.size() != buckets) - refuse("bucket limits"); + refuse_policy("bucket limits"); } bool due(const Policy& policy, const State& state, const double z) { @@ -214,11 +216,11 @@ Monitor::Monitor(Config config) : config_(std::move(config)) { policy.launch_floor = 0.0; validate(policy); if (!(config_.floor_share >= 0.0) || !(config_.floor_margin >= 0.0)) - refuse("floor share or margin"); + refuse_policy("floor share or margin"); if (!(config_.budget > 0.0)) - refuse("budget"); + refuse_policy("budget"); if (config_.rate.empty()) - refuse("collar rate: the coarse axis has no samples"); + refuse_policy("collar rate: the coarse axis has no samples"); // `_busy` starts true: every step is checked until the launch says otherwise. state_.busy = true; } diff --git a/native/tests/native/execution.hpp b/native/tests/native/execution.hpp index e61a02244..cc95a94c9 100644 --- a/native/tests/native/execution.hpp +++ b/native/tests/native/execution.hpp @@ -15,7 +15,7 @@ namespace solprop_native::test { inline CpuExecution make_table() { CpuExecution table; -#if defined(__APPLE__) +#ifdef __APPLE__ constexpr std::size_t stage = 8192; constexpr std::size_t fft = 32768; constexpr std::size_t fft_thread = 32768; @@ -30,7 +30,8 @@ inline CpuExecution make_table() { #endif const auto set = [&](CpuOperation operation, std::size_t threshold, std::size_t full) { - table.grain[static_cast(operation)] = {threshold, full}; + table.grain[static_cast(operation)] = {.threshold = threshold, + .full = full}; }; set(CpuOperation::stage, stage, stage); set(CpuOperation::response, 32768, 196608); diff --git a/native/tests/native/main.cpp b/native/tests/native/main.cpp index 7d9a924f1..8dab7d460 100644 --- a/native/tests/native/main.cpp +++ b/native/tests/native/main.cpp @@ -3,7 +3,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include - #include #include diff --git a/native/tests/native/test_adams_march.cpp b/native/tests/native/test_adams_march.cpp index 08603955d..e588e8e5b 100644 --- a/native/tests/native/test_adams_march.cpp +++ b/native/tests/native/test_adams_march.cpp @@ -10,6 +10,8 @@ #include "solprop_native/adams_march.hpp" +#include + #include #include #include @@ -17,8 +19,6 @@ #include #include -#include - namespace { namespace march = solprop_native::march; diff --git a/native/tests/native/test_adams_phi.cpp b/native/tests/native/test_adams_phi.cpp index 11de12f3a..faf33b3ac 100644 --- a/native/tests/native/test_adams_phi.cpp +++ b/native/tests/native/test_adams_phi.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -92,7 +93,7 @@ void add_threshold(std::vector& out, const double t) { for (const double r : {std::nextafter(t, 0.0), t, std::nextafter(t, infinity)}) { out.emplace_back(r, 0.0); out.emplace_back(0.0, -r); - const double d = r / std::sqrt(2.0); + const double d = r / std::numbers::sqrt2; for (const double e : {std::nextafter(d, 0.0), d, std::nextafter(d, infinity)}) out.emplace_back(-e, e); } @@ -143,6 +144,7 @@ TEST(AdamsPhiLanes, EdgeArgumentsAreTheScalarFormsBits) { TEST(AdamsPhiLanes, EachLaneIsItsOwnWhateverItsNeighbours) { // Mixed blocks of every length: small, series, near-threshold and large arguments // side by side, so that lanes stop their series, square and branch apart. + // NOLINTNEXTLINE(bugprone-random-generator-seed): a fixed seed keeps the test reproducible std::mt19937_64 generator(20261029); std::uniform_real_distribution phase(-3.2, 3.2); std::uniform_real_distribution exponent(-12.0, 1.2); diff --git a/native/tests/native/test_aperture.cpp b/native/tests/native/test_aperture.cpp index a518b1842..cb7184ad5 100644 --- a/native/tests/native/test_aperture.cpp +++ b/native/tests/native/test_aperture.cpp @@ -15,7 +15,6 @@ #include "solprop_native/far_field.hpp" #include - #include #include @@ -69,18 +68,19 @@ Case make_case(const std::size_t n_x, const std::size_t n_y, const std::size_t n Case data; auto& problem = data.problem; const std::size_t n = 9; - problem.n_x = n_x; - problem.n_y = n_y; - problem.n_omega = n; - problem.n_t = envelope ? n : 2 * (n - 1); - problem.n_pol_state = n_pol; - problem.n_modes = n_x * n_y; - problem.dt = 1.5e-15; - problem.family = + problem.spatial.n_x = n_x; + problem.spatial.n_y = n_y; + problem.dims.n_omega = n; + problem.dims.n_t = envelope ? n : 2 * (n - 1); + problem.dims.n_pol_state = n_pol; + problem.dims.n_modes = n_x * n_y; + problem.grid.dt = 1.5e-15; + problem.dims.family = envelope ? native::FieldFamily::envelope : native::FieldFamily::field_real; - problem.spatial_k_x = fft_axis(n_x, n_y > 1 ? 1.5e6 : 1.0e5); - problem.spatial_k_y = n_y > 1 ? fft_axis(n_y, 1.2e6) : std::vector{0.0}; + problem.spatial.k_x = fft_axis(n_x, n_y > 1 ? 1.5e6 : 1.0e5); + problem.spatial.k_y = n_y > 1 ? fft_axis(n_y, 1.2e6) : std::vector{0.0}; const std::size_t columns = n_x * n_y; + // NOLINTNEXTLINE(bugprone-random-generator-seed): a fixed seed keeps the test reproducible std::mt19937_64 random(7); std::normal_distribution normal; data.field.resize(columns * n_pol * n); @@ -91,8 +91,8 @@ Case make_case(const std::size_t n_x, const std::size_t n_y, const std::size_t n const double phase = 0.3 * static_cast(column); data.on_axis.emplace_back(std::cos(phase) / static_cast(columns), std::sin(phase) / static_cast(columns)); - const double kx = problem.spatial_k_x[column % n_x]; - const double ky = problem.spatial_k_y[column / n_x]; + const double kx = problem.spatial.k_x[column % n_x]; + const double ky = problem.spatial.k_y[column / n_x]; data.transverse.push_back(kx * kx + ky * ky); data.transverse_low.push_back(1e-3 * static_cast(column % 3)); data.transverse_rate.push_back(column % 2 == 0 ? 0.0 : 3.0); @@ -152,13 +152,13 @@ template auto device(const char* label, const std::vector& values) // One row's table entry, by its constant. double& constant_of(Case& data, const std::size_t index, const native::FarFieldConstant constant) { - const auto row = native::far_field_aperture_constants + data.problem.n_omega; + const auto row = native::far_field_aperture_constants + data.problem.dims.n_omega; return data.apertures[index * row + constant]; } std::vector run(Case& data, const std::uint64_t flags, const int profile, const int exponent) { - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; // Every row takes `profile`; a test of mixed rows passes kPerRow and sets its own. std::vector apertures = data.apertures; if (profile != kPerRow) { @@ -166,7 +166,8 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile for (std::size_t index = 0; index * row < apertures.size(); ++index) apertures[index * row + native::far_field_profile] = static_cast(profile); } - const auto rows = data.problem.n_x * data.problem.n_y * data.problem.n_pol_state; + const auto rows = data.problem.spatial.n_x * data.problem.spatial.n_y * + data.problem.dims.n_pol_state; native::FarFieldPass::ComplexView2 field("test.field", rows, n); const auto host = Kokkos::create_mirror_view(field); for (std::size_t row = 0; row < rows; ++row) @@ -175,18 +176,18 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile Kokkos::deep_copy(field, host); native::FarFieldPass pass(data.problem, native::ExecutionSpace{}); const native::FarFieldPass::Inputs inputs{ - device("test.weights", data.weights), - device("test.on_axis", data.on_axis), - device("test.square", data.square), - device("test.square_low", data.square_low), - device("test.transverse", data.transverse), - device("test.transverse_low", data.transverse_low), - device("test.spectral_rate", data.spectral_rate), - device("test.user_rate", data.user_rate), - device("test.transverse_rate", data.transverse_rate), - device("test.omega", data.omega), - device("test.active", data.active), - data.evanescent}; + .weights = device("test.weights", data.weights), + .on_axis = device("test.on_axis", data.on_axis), + .square = device("test.square", data.square), + .square_low = device("test.square_low", data.square_low), + .transverse = device("test.transverse", data.transverse), + .transverse_low = device("test.transverse_low", data.transverse_low), + .spectral_rate = device("test.spectral_rate", data.spectral_rate), + .user_rate = device("test.user_rate", data.user_rate), + .transverse_rate = device("test.transverse_rate", data.transverse_rate), + .omega = device("test.omega", data.omega), + .active = device("test.active", data.active), + .evanescent = data.evanescent}; auto values = pass.run(flags, apertures, exponent, data.regions, 0.5, field, inputs); pass.close(); return values; @@ -197,7 +198,7 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile // in TheShapesAreTheirFormulas. double host_weight(const Case& data, const std::size_t index, const double kx, const double ky, const std::size_t bin, const int profile) { - const auto row = native::far_field_aperture_constants + data.problem.n_omega; + const auto row = native::far_field_aperture_constants + data.problem.dims.n_omega; const double* constants = &data.apertures[index * row]; const double scale = constants[native::far_field_aperture_constants + bin]; const double centre_x = constants[native::far_field_centre_x]; @@ -219,12 +220,13 @@ double host_weight(const Case& data, const std::size_t index, const double kx, return native::far_field_shaped_weight( native::far_field_offset(scale, kx, centre_x), native::far_field_offset(scale, ky, centre_y), - native::FarFieldOutline{constants[native::far_field_half_width], - constants[native::far_field_half_height], - constants[native::far_field_stretch_x], - constants[native::far_field_stretch_y], - constants[native::far_field_inverse_width], - constants[native::far_field_edge]}, + native::FarFieldOutline{.half_width = constants[native::far_field_half_width], + .half_height = constants[native::far_field_half_height], + .stretch_x = constants[native::far_field_stretch_x], + .stretch_y = constants[native::far_field_stretch_y], + .inverse_width = + constants[native::far_field_inverse_width], + .edge = constants[native::far_field_edge]}, shape, code); } @@ -234,27 +236,31 @@ double host_weight(const Case& data, const std::size_t index, const double kx, std::vector reference(const Case& data, const int profile, const int exponent, const bool magnitude) { const auto& problem = data.problem; - const auto n = problem.n_omega; - const auto n_pol = problem.n_pol_state; - const auto columns = problem.n_x * problem.n_y; - const bool envelope = problem.family == native::FieldFamily::envelope; - const double dt = problem.dt; + const auto n = problem.dims.n_omega; + const auto n_pol = problem.dims.n_pol_state; + const auto columns = problem.spatial.n_x * problem.spatial.n_y; + const bool envelope = problem.dims.family == native::FieldFamily::envelope; + const double dt = problem.grid.dt; const double density = envelope ? dt * dt / (2.0 * std::numbers::pi) : 2.0 * dt * dt / std::numbers::pi; const std::size_t row = native::far_field_aperture_constants + n; const std::size_t n_apertures = data.apertures.size() / row; const std::size_t n_regions = data.regions.size() / 4; - std::vector beam(n), absorbed(native::far_field_absorbed_channels), axis(n); + std::vector beam(n); + std::vector absorbed(native::far_field_absorbed_channels); + std::vector axis(n); std::vector> regions(n_regions, std::vector(n)); std::vector> collected(n_apertures, std::vector(n)); std::vector> through(n_apertures, std::vector(n)); for (std::size_t bin = 0; bin < n; ++bin) { // Each amplitude and its magnetic closure, the same sum with conj(eta) applied. - std::vector> amplitude(n_pol), axial(n_apertures * n_pol); - std::vector> closed(n_pol), closed_axial(n_apertures * n_pol); + std::vector> amplitude(n_pol); + std::vector> axial(n_apertures * n_pol); + std::vector> closed(n_pol); + std::vector> closed_axial(n_apertures * n_pol); for (std::size_t column = 0; column < columns; ++column) { - const double kx = problem.spatial_k_x[column % problem.n_x]; - const double ky = problem.spatial_k_y[column / problem.n_x]; + const double kx = problem.spatial.k_x[column % problem.spatial.n_x]; + const double ky = problem.spatial.k_y[column / problem.spatial.n_x]; double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) power += std::norm( @@ -279,7 +285,7 @@ std::vector reference(const Case& data, const int profile, const int exp const double weighted = data.weights[column] * power * closing.real(); beam[bin] += weighted; const double energy = (envelope || bin == 0 || bin + 1 == n ? 1.0 : 4.0) * dt / - static_cast(problem.n_t); + static_cast(problem.dims.n_t); absorbed[0] += energy * weighted * (data.spectral_rate[bin] - data.user_rate[bin]); absorbed[4] += energy * weighted * data.user_rate[bin]; @@ -327,6 +333,7 @@ std::vector reference(const Case& data, const int profile, const int exp .real(); } std::vector packed; + packed.reserve(beam.size()); for (const double value : beam) packed.push_back(0.5 * density * value); for (const double value : absorbed) @@ -381,7 +388,7 @@ TEST_P(FarFieldPassAgainstHost, EveryStatisticMatchesTheSerialReduction) { expect_matches(actual, expected, magnitude, variant.n_x * variant.n_y); // Every channel but the evanescent one, which carries no axial flux, and every // aperture take part on this fixture. - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; for (const std::size_t channel : {0U, 1U, 3U, 4U}) EXPECT_NE(actual[n + channel], 0.0) << channel; EXPECT_EQ(actual[n + 2], 0.0); @@ -406,7 +413,8 @@ TEST(FarFieldPass, EveryRadiusPathMatchesTheSerialReduction) { // and a second distance (neither): each against the host reduction. for (const int path : {0, 1, 2}) { auto data = make_case(5, 4, 2, true, 1); - const std::size_t row = native::far_field_aperture_constants + data.problem.n_omega; + const std::size_t row = + native::far_field_aperture_constants + data.problem.dims.n_omega; for (std::size_t index = 1; index < 3; ++index) { if (path == 0) { data.apertures[index * row + native::far_field_centre_x] = @@ -416,7 +424,7 @@ TEST(FarFieldPass, EveryRadiusPathMatchesTheSerialReduction) { } if (path == 2 && index == 2) { data.apertures[index * row + native::far_field_distance] = 0.3; - for (std::size_t bin = 0; bin < data.problem.n_omega; ++bin) + for (std::size_t bin = 0; bin < data.problem.dims.n_omega; ++bin) data.apertures[index * row + native::far_field_aperture_constants + bin] = 0.3 * native::far_field_speed_of_light / data.omega[bin]; } @@ -432,7 +440,7 @@ TEST(FarFieldPass, MixedShapesAndProfilesMatchTheSerialReduction) { // ellipse, one distance and then two, so the shaped branch runs under both origins. for (const bool one_distance : {true, false}) { auto data = make_case(5, 4, 2, true, 1); - const std::size_t n = data.problem.n_omega; + const std::size_t n = data.problem.dims.n_omega; const std::size_t row = native::far_field_aperture_constants + n; const auto scale_row = std::vector(data.apertures.begin() + @@ -480,7 +488,7 @@ TEST(FarFieldPass, AHalfPlaneAndItsComplementTileTheColumns) { // In a band whose outer columns the map still reaches, both rows at one edge sum to // the terms of every column inside the disc: exactly one row holds each, by one sign. auto data = make_case(5, 4, 1, true, 0); - const std::size_t n = data.problem.n_omega; + const std::size_t n = data.problem.dims.n_omega; for (const auto shape_pair : {std::array{native::far_field_half_plane_x_ge, native::far_field_half_plane_x_lt}, std::array{native::far_field_half_plane_y_ge, @@ -495,11 +503,11 @@ TEST(FarFieldPass, AHalfPlaneAndItsComplementTileTheColumns) { constant_of(rows, index, native::far_field_centre_y) = shape_pair[0] == native::far_field_half_plane_x_ge ? 0.0 : -0.02; } - const auto columns = rows.problem.n_x * rows.problem.n_y; + const auto columns = rows.problem.spatial.n_x * rows.problem.spatial.n_y; for (std::size_t column = 0; column < columns; ++column) for (std::size_t bin = 0; bin < n; ++bin) { - const double kx = rows.problem.spatial_k_x[column % rows.problem.n_x]; - const double ky = rows.problem.spatial_k_y[column / rows.problem.n_x]; + const double kx = rows.problem.spatial.k_x[column % rows.problem.spatial.n_x]; + const double ky = rows.problem.spatial.k_y[column / rows.problem.spatial.n_x]; const double first = host_weight(rows, 0, kx, ky, bin, 0); const double second = host_weight(rows, 1, kx, ky, bin, 0); const double scale = rows.apertures[native::far_field_aperture_constants + bin]; @@ -515,9 +523,12 @@ TEST(FarFieldPass, TheShapesAreTheirFormulas) { const auto outline = [](const double half_width, const double half_height, const double stretch_x, const double stretch_y, const double width) { - return native::FarFieldOutline{ - half_width, half_height, stretch_x, stretch_y, width > 0.0 ? 1.0 / width : 0.0, - width}; + return native::FarFieldOutline{.half_width = half_width, + .half_height = half_height, + .stretch_x = stretch_x, + .stretch_y = stretch_y, + .inverse_width = width > 0.0 ? 1.0 / width : 0.0, + .edge = width}; }; // The offset is X - x, so x >= X reads across <= 0, whichever side is asked for. EXPECT_EQ(native::far_field_offset(2.0, -3.0, 1.0), -5.0); @@ -598,7 +609,7 @@ TEST(FarFieldPass, TheShapesAreTheirFormulas) { TEST(FarFieldPass, APartialRequestPacksOnlyItsRows) { auto data = make_case(5, 4, 1, true, 1); const auto every = run(data, kEvery, 1, 2); - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; // The beam spectrum and the apertures' on-axis spectra alone: the same values as // the full request's rows, within the bound (another slot count, the same sums). const auto partial = @@ -632,7 +643,7 @@ TEST(FarFieldPass, DeclaredAperturesCostNothingUnlessAsked) { data.apertures.clear(); const auto bare = run(data, flags, 2, 1); ASSERT_EQ(declared.size(), bare.size()); - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; constexpr auto absorbed = native::far_field_absorbed_channels; ASSERT_EQ(bare.size(), (1 + 4 + 1) * n + absorbed); const double relative = 3.0 * gamma(8 * 20 + 64) + 8.0 * kEpsilon; @@ -782,7 +793,7 @@ TEST(FarFieldPass, InvalidRequestsAreRefused) { EXPECT_THROW(static_cast(run(bare, native::far_field_collected, 1, 2)), std::invalid_argument); auto radial = data; - radial.problem.n_x = 0; + radial.problem.spatial.n_x = 0; EXPECT_THROW(static_cast(run(radial, kEvery, 1, 2)), std::invalid_argument); } diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 2e2446c37..9382fae59 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -2,15 +2,13 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "bounds.hpp" +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/gemm.hpp" #include "solprop_native/plasma.hpp" -#include "bounds.hpp" -#include "execution.hpp" - #include - #include #include @@ -26,7 +24,7 @@ #include #include -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP #include #endif @@ -95,121 +93,127 @@ std::vector real_time(const std::vector& spectrum, native::Problem base_problem(const native::FieldFamily family) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = family == native::FieldFamily::envelope ? 4 : 4; - problem.n_to = family == native::FieldFamily::envelope ? 4 : 8; - problem.n_omega = family == native::FieldFamily::envelope ? 4 : 3; - problem.n_stages = 1; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.family = family; - problem.crop_index = {0, 1, 2}; + problem.dims.n_t = 4; + problem.dims.n_to = family == native::FieldFamily::envelope ? 4 : 8; + problem.dims.n_omega = family == native::FieldFamily::envelope ? 4 : 3; + problem.stepper.n_stages = 1; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.family = family; + problem.grid.crop_index = {0, 1, 2}; if (family == native::FieldFamily::envelope) - problem.crop_index.push_back(3); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.omega.push_back(static_cast(index + 1)); - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_omega, Complex(1.0, 0.0)); - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.25}; - problem.synthesis = 2.0; - problem.projection = 0.5; - problem.kerr_coefficient = 0.125; - problem.self_factor = 0.5; - problem.conjugate_factor = 0.25; - problem.harmonic_factor = 0.0; - problem.take_real = family != native::FieldFamily::envelope; + problem.grid.crop_index.push_back(3); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.grid.omega.push_back(static_cast(index + 1)); + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_omega, Complex(1.0, 0.0)); + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.25}; + problem.transform.synthesis = 2.0; + problem.transform.projection = 0.5; + problem.kerr.coefficient = 0.125; + problem.kerr.self_factor = 0.5; + problem.kerr.conjugate_factor = 0.25; + problem.kerr.harmonic_factor = 0.0; + problem.kerr.take_real = family != native::FieldFamily::envelope; return problem; } native::TrialScalars trial(const native::Problem& problem, const double taper = 1.0) { return { - std::vector((problem.n_stages + 1) * problem.n_phi_terms, 0.0), - std::vector(problem.n_stages * problem.n_phi_terms, 0.0), - std::vector(problem.n_stages * problem.n_prefactor_terms, 1.0), - std::vector(problem.n_stages, 1.0), - std::vector(problem.n_stages, taper), - {}, - std::vector(problem.n_stages * problem.n_raman_terms, 0.0), - {}, - {}, - {}, - {}, - 0, - {}, - {}, - {}, + .phi_delta = std::vector( + (problem.stepper.n_stages + 1) * problem.linear.n_phi_terms, 0.0), + .phi_at = std::vector( + problem.stepper.n_stages * problem.linear.n_phi_terms, 0.0), + .prefactor = std::vector( + problem.stepper.n_stages * problem.linear.n_prefactor_terms, 1.0), + .density = std::vector(problem.stepper.n_stages, 1.0), + .taper = std::vector(problem.stepper.n_stages, taper), + .preionisation = {}, + .raman_rates = + std::vector(problem.stepper.n_stages * problem.raman.n_terms, 0.0), + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}, }; } native::Problem multimode_problem(const std::size_t node_chunk = 2) { auto problem = base_problem(native::FieldFamily::field_real); - problem.n_modes = 2; - problem.n_pol_state = 1; - problem.n_pol_transform = 1; - problem.n_nodes = 2; - problem.node_chunk = node_chunk; - problem.nonlinear = true; - problem.synthesis_matrix = {1.0, 0.5, -0.25, 1.5}; - problem.projection_matrix = {0.75, -0.5, 0.25, 1.25}; - problem.projection_coarse_matrix = {0.5, -0.25, 0.125, 1.0}; - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_modes = 2; + problem.dims.n_pol_state = 1; + problem.dims.n_pol_transform = 1; + problem.dims.n_nodes = 2; + problem.dims.node_chunk = node_chunk; + problem.kerr.nonlinear = true; + problem.transform.synthesis_matrix = {1.0, 0.5, -0.25, 1.5}; + problem.transform.projection_matrix = {0.75, -0.5, 0.25, 1.25}; + problem.transform.projection_coarse_matrix = {0.5, -0.25, 0.125, 1.0}; + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); return problem; } native::Problem general_problem(const native::FieldFamily family) { auto problem = base_problem(family); - problem.nonlinear = true; - problem.synthesis_matrix = {problem.synthesis}; - problem.projection_matrix = {problem.projection}; + problem.kerr.nonlinear = true; + problem.transform.synthesis_matrix = {problem.transform.synthesis}; + problem.transform.projection_matrix = {problem.transform.projection}; return problem; } native::Problem plasma_problem() { auto problem = general_problem(native::FieldFamily::field_real); - problem.nonlinear = false; - problem.plasma = true; - problem.stash_plasma = true; - problem.dt_fine = 0.125; - problem.charge_mass_ratio = 2.0; - problem.ionisation_potential = 3.0; - problem.log_field_min = std::log(0.01); - problem.d_log_field = std::log(1000.0); - problem.field_min = 0.01; - problem.field_max = 10.0; - problem.rate_max = 0.5; - problem.rate_intervals = 1; - problem.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; + problem.kerr.nonlinear = false; + problem.plasma.enabled = true; + problem.plasma.stash = true; + problem.grid.dt_fine = 0.125; + problem.plasma.charge_mass_ratio = 2.0; + problem.plasma.ionisation_potential = 3.0; + problem.plasma.log_field_min = std::log(0.01); + problem.plasma.d_log_field = std::log(1000.0); + problem.plasma.field_min = 0.01; + problem.plasma.field_max = 10.0; + problem.plasma.rate_max = 0.5; + problem.plasma.rate_intervals = 1; + problem.plasma.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; return problem; } native::Problem raman_problem(const native::FieldFamily family) { auto problem = general_problem(family); - problem.nonlinear = false; - problem.raman = true; - problem.raman_thg = family == native::FieldFamily::field_real; - problem.n_raman_terms = 2; - problem.raman_tensor_weights = {1.0, 0.0, 1.0, 0.0}; - problem.raman_fraction = 0.4; - problem.raman_epsilon0 = 2.0; - problem.dt_fine = 0.125; - problem.raman_undamped.assign(problem.n_raman_terms * 2 * problem.n_to, 0.0); - problem.raman_undamped[1] = 1.0; - problem.raman_undamped[2] = 0.5; - problem.raman_undamped[3] = -0.25; - const auto second = 2 * problem.n_to; - problem.raman_undamped[second + 1] = 0.2; - problem.raman_undamped[second + 2] = 0.1; - problem.raman_undamped[second + 3] = 0.05; + problem.kerr.nonlinear = false; + problem.raman.enabled = true; + problem.raman.thg = family == native::FieldFamily::field_real; + problem.raman.n_terms = 2; + problem.raman.tensor_weights = {1.0, 0.0, 1.0, 0.0}; + problem.raman.fraction = 0.4; + problem.raman.epsilon0 = 2.0; + problem.grid.dt_fine = 0.125; + problem.raman.undamped.assign(problem.raman.n_terms * 2 * problem.dims.n_to, 0.0); + problem.raman.undamped[1] = 1.0; + problem.raman.undamped[2] = 0.5; + problem.raman.undamped[3] = -0.25; + const auto second = 2 * problem.dims.n_to; + problem.raman.undamped[second + 1] = 0.2; + problem.raman.undamped[second + 2] = 0.1; + problem.raman.undamped[second + 3] = 0.05; return problem; } @@ -217,94 +221,95 @@ std::vector raman_reference(const native::Problem& problem, const std::vector& spectrum, const std::vector& rates) { std::vector modal; - if (problem.family == native::FieldFamily::field_real) { - for (const double value : real_time(spectrum, problem.n_to)) + if (problem.dims.family == native::FieldFamily::field_real) { + for (const double value : real_time(spectrum, problem.dims.n_to)) modal.emplace_back(value, 0.0); - } else if (problem.family == native::FieldFamily::field_analytic) { - auto analytic = std::vector(problem.n_to); + } else if (problem.dims.family == native::FieldFamily::field_analytic) { + auto analytic = std::vector(problem.dims.n_to); analytic[0] = spectrum[0]; for (std::size_t index = 1; index < spectrum.size(); ++index) analytic[index] = 2.0 * spectrum[index]; - modal = envelope_time(analytic, problem.n_to); + modal = envelope_time(analytic, problem.dims.n_to); } else { - modal = envelope_time(spectrum, problem.n_to); + modal = envelope_time(spectrum, problem.dims.n_to); } - const auto doubled = 2 * problem.n_to; + const auto doubled = 2 * problem.dims.n_to; std::vector kernel(doubled); for (std::size_t time = 0; time < doubled; ++time) { double total = 0.0; - const double delay = static_cast(time) * problem.dt_fine; - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) - total += problem.raman_undamped[term * doubled + time] * + const double delay = static_cast(time) * problem.grid.dt_fine; + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) + total += problem.raman.undamped[term * doubled + time] * std::exp(-rates[term] * delay); - kernel[time] = problem.raman_fraction * total; + kernel[time] = problem.raman.fraction * total; } - std::vector projected(problem.n_to); - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - const double synthesis = problem.synthesis_matrix[node]; - const double projection = problem.projection_matrix[node]; - std::vector field(problem.n_to); - std::vector drive(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::vector projected(problem.dims.n_to); + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + const double synthesis = problem.transform.synthesis_matrix[node]; + const double projection = problem.transform.projection_matrix[node]; + std::vector field(problem.dims.n_to); + std::vector drive(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { field[time] = synthesis * modal[time]; - drive[time] = problem.raman_thg ? field[time].real() * field[time].real() + drive[time] = problem.raman.thg ? field[time].real() * field[time].real() : 0.5 * (field[time].real() * field[time].real() + field[time].imag() * field[time].imag()); } - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double convolved = 0.0; for (std::size_t previous = 0; previous <= time; ++previous) convolved += drive[previous] * kernel[time - previous]; - convolved *= problem.dt_fine; - const Complex multiplier = problem.family == native::FieldFamily::envelope + convolved *= problem.grid.dt_fine; + const Complex multiplier = problem.dims.family == native::FieldFamily::envelope ? field[time] : Complex(field[time].real(), 0.0); - projected[time] += projection * problem.raman_epsilon0 * multiplier * convolved * - problem.time_window[time]; + projected[time] += projection * problem.raman.epsilon0 * multiplier * convolved * + problem.grid.time_window[time]; } } const auto transformed = dft(projected); return {transformed.begin(), - transformed.begin() + static_cast(problem.n_omega)}; + transformed.begin() + static_cast(problem.dims.n_omega)}; } std::vector multimode_reference(const native::Problem& problem, const std::vector& spectrum, const bool coarse = false) { - std::vector> modal(problem.n_modes); - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) { + std::vector> modal(problem.dims.n_modes); + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) { const auto first = - spectrum.begin() + static_cast(mode * problem.n_omega); + spectrum.begin() + static_cast(mode * problem.dims.n_omega); const std::vector one( - first, first + static_cast(problem.n_omega)); - modal[mode] = real_time(one, problem.n_to); + first, first + static_cast(problem.dims.n_omega)); + modal[mode] = real_time(one, problem.dims.n_to); } - std::vector> polarisation(problem.n_nodes, - std::vector(problem.n_to)); - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::vector> polarisation(problem.dims.n_nodes, + std::vector(problem.dims.n_to)); + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double value = 0.0; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) value += - problem.synthesis_matrix[node * problem.n_modes + mode] * modal[mode][time]; - polarisation[node][time] = problem.kerr_coefficient * value * value * value; + problem.transform.synthesis_matrix[node * problem.dims.n_modes + mode] * + modal[mode][time]; + polarisation[node][time] = problem.kerr.coefficient * value * value * value; } } - std::vector result(problem.n_modes * problem.n_omega); - const auto& projection = - coarse ? problem.projection_coarse_matrix : problem.projection_matrix; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) { - std::vector projected(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) { - for (std::size_t node = 0; node < problem.n_nodes; ++node) + std::vector result(problem.dims.n_modes * problem.dims.n_omega); + const auto& projection = coarse ? problem.transform.projection_coarse_matrix + : problem.transform.projection_matrix; + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) { + std::vector projected(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) projected[time] += - projection[mode * problem.n_nodes + node] * polarisation[node][time]; + projection[mode * problem.dims.n_nodes + node] * polarisation[node][time]; } const auto transformed = dft(projected); - for (std::size_t index = 0; index < problem.n_omega; ++index) - result[mode * problem.n_omega + index] = transformed[index]; + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + result[mode * problem.dims.n_omega + index] = transformed[index]; } return result; } @@ -313,38 +318,40 @@ std::vector nonlinear_reference(const native::Problem& problem, const std::vector& spectrum, const double taper) { std::vector time; - if (problem.family == native::FieldFamily::envelope) { - time = envelope_time(spectrum, problem.n_to); - } else if (problem.family == native::FieldFamily::field_real) { - const auto real = real_time(spectrum, problem.n_to); + if (problem.dims.family == native::FieldFamily::envelope) { + time = envelope_time(spectrum, problem.dims.n_to); + } else if (problem.dims.family == native::FieldFamily::field_real) { + const auto real = real_time(spectrum, problem.dims.n_to); for (const auto value : real) time.emplace_back(value, 0.0); } else { - std::vector analytic(problem.n_to); + std::vector analytic(problem.dims.n_to); analytic[0] = spectrum[0]; for (std::size_t index = 1; index < spectrum.size(); ++index) { analytic[index] = 2.0 * spectrum[index]; } - time = envelope_time(analytic, problem.n_to); + time = envelope_time(analytic, problem.dims.n_to); } - std::vector projected(problem.n_to); - for (std::size_t index = 0; index < problem.n_to; ++index) { - const Complex node = problem.synthesis * taper * time[index]; + std::vector projected(problem.dims.n_to); + for (std::size_t index = 0; index < problem.dims.n_to; ++index) { + const Complex node = problem.transform.synthesis * taper * time[index]; Complex polarisation; - if (problem.family == native::FieldFamily::field_real) { - polarisation = problem.kerr_coefficient * node.real() * node.real() * + if (problem.dims.family == native::FieldFamily::field_real) { + polarisation = problem.kerr.coefficient * node.real() * node.real() * Complex(node.real(), 0.0); } else { const double modulus = node.real() * node.real() + node.imag() * node.imag(); - polarisation = problem.kerr_coefficient * - (problem.self_factor + problem.conjugate_factor) * modulus * node; + polarisation = problem.kerr.coefficient * + (problem.kerr.self_factor + problem.kerr.conjugate_factor) * + modulus * node; } - polarisation *= problem.projection / taper; + polarisation *= problem.transform.projection / taper; projected[index] = - problem.take_real ? Complex(polarisation.real(), 0.0) : polarisation; + problem.kerr.take_real ? Complex(polarisation.real(), 0.0) : polarisation; } const auto full = dft(projected); - return {full.begin(), full.begin() + static_cast(problem.n_omega)}; + return {full.begin(), + full.begin() + static_cast(problem.dims.n_omega)}; } void expect_complex_near(const Complex actual, const Complex expected, @@ -404,19 +411,19 @@ TEST(MultiplyPhase, AppliesQuarterTurnAndPreservesMagnitude) { TEST(GeneralState, MasksAndAppliesCombinedWholeStepPropagator) { auto problem = general_problem(native::FieldFamily::envelope); - problem.n_phi_terms = 2; - problem.phi_terms.clear(); + problem.linear.n_phi_terms = 2; + problem.linear.phi_terms.clear(); for (std::size_t term = 0; term < 2; ++term) { - for (std::size_t index = 0; index < problem.n_omega; ++index) { - problem.phi_terms.emplace_back(0.0, static_cast(term + 1) * - static_cast(index + 1)); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + problem.linear.phi_terms.emplace_back(0.0, static_cast(term + 1) * + static_cast(index + 1)); } } - problem.nonlinear = false; - problem.active_mask[2] = 0U; + problem.kerr.nonlinear = false; + problem.grid.active_mask[2] = 0U; auto scalars = trial(problem); scalars.phi_delta = {0.1, 0.2, 0.3, 0.4}; - const std::vector initial(problem.n_omega, {1.0, -0.5}); + const std::vector initial(problem.dims.n_omega, {1.0, -0.5}); native::GeneralState state(problem); state.set_initial(initial, {}); const auto partials = state.trial_step(0.25, scalars); @@ -439,12 +446,12 @@ TEST(GeneralState, KerrFamiliesMatchStraightforwardHostTransforms) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = general_problem(family); - problem.nonlinear = true; - std::vector initial(problem.n_omega); + problem.kerr.nonlinear = true; + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); constexpr double taper = 1.5; constexpr double dz = 0.02; @@ -459,7 +466,7 @@ TEST(GeneralState, KerrFamiliesMatchStraightforwardHostTransforms) { const Complex expected = initial[index] + dz * derivative[index]; magnitude = std::max(magnitude, Kokkos::abs(expected)); const double tolerance = - 512.0 * kEpsilon * static_cast(problem.n_to) * magnitude; + 512.0 * kEpsilon * static_cast(problem.dims.n_to) * magnitude; expect_complex_near(result[index], expected, tolerance); } EXPECT_GT(partials[2], 0.0); @@ -477,7 +484,7 @@ TEST(GeneralState, MultimodeTransformMatchesStraightforwardHostImplementation) { EXPECT_THROW(static_cast(state.stage_to_host(0)), std::logic_error); const auto partials = state.trial_step(dz, trial(problem)); const auto stage = state.stage_to_host(0); - EXPECT_THROW(static_cast(state.stage_to_host(problem.n_stages)), + EXPECT_THROW(static_cast(state.stage_to_host(problem.stepper.n_stages)), std::out_of_range); ASSERT_EQ(stage.size(), derivative.size()); for (std::size_t index = 0; index < stage.size(); ++index) { @@ -487,7 +494,7 @@ TEST(GeneralState, MultimodeTransformMatchesStraightforwardHostImplementation) { } state.finish(dz); const auto result = state.field_to_host(); - ASSERT_EQ(partials.size(), 3 * problem.n_modes); + ASSERT_EQ(partials.size(), 3 * problem.dims.n_modes); ASSERT_EQ(result.size(), initial.size()); for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; @@ -518,7 +525,7 @@ TEST(GeneralState, NodeChunkingPreservesTheMultimodeResult) { TEST(GeneralState, QuadratureErrorUsesFineAndCoarseProjectionOfTheSameResponse) { auto problem = multimode_problem(); - problem.first_same_as_last = false; + problem.stepper.first_same_as_last = false; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; const auto fine = multimode_reference(problem, initial); @@ -536,18 +543,19 @@ TEST(GeneralState, QuadratureErrorUsesFineAndCoarseProjectionOfTheSameResponse) static_cast(state.trial_step(0.01, trial(problem))); state.promote_fsal(); EXPECT_NEAR(state.quadrature_error(), expected, - 512.0 * static_cast(problem.n_to) * kEpsilon); + 512.0 * static_cast(problem.dims.n_to) * kEpsilon); } TEST(GeneralState, WeightedResidualDividesEachBinBeforeReducing) { auto problem = multimode_problem(); - problem.weighted_controller = true; + problem.stepper.weighted_controller = true; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; native::GeneralState state(problem); state.set_initial(initial, {}); const auto ordinary = state.trial_step(0.01, trial(problem)); - const auto weighted = state.weighted_error(std::vector(problem.n_omega, 2.0)); + const auto weighted = + state.weighted_error(std::vector(problem.dims.n_omega, 2.0)); ASSERT_EQ(weighted.size(), problem.n_columns()); for (std::size_t column = 0; column < problem.n_columns(); ++column) EXPECT_NEAR(weighted[column], ordinary[2 * problem.n_columns() + column] / 4.0, @@ -556,19 +564,19 @@ TEST(GeneralState, WeightedResidualDividesEachBinBeforeReducing) { TEST(GeneralState, PackedStatisticsMatchStraightforwardHostReductions) { auto problem = multimode_problem(); - problem.family = native::FieldFamily::envelope; - problem.n_t = 4; - problem.n_to = 4; - problem.n_omega = 4; - problem.crop_index = {0, 1, 2, 3}; - problem.active_mask.assign(4, 1U); - problem.norm_mask.assign(4, 1U); - problem.omega_window.assign(4, 1.0); - problem.omega = {1.0, 2.0, 3.0, 4.0}; - problem.time_window.assign(4, 1.0); - problem.carrier.assign(4, Complex(1.0, 0.0)); - problem.phi_terms.assign(8, Complex{}); - problem.prefactor_terms.assign(8, Complex(1.0, 0.0)); + problem.dims.family = native::FieldFamily::envelope; + problem.dims.n_t = 4; + problem.dims.n_to = 4; + problem.dims.n_omega = 4; + problem.grid.crop_index = {0, 1, 2, 3}; + problem.grid.active_mask.assign(4, 1U); + problem.grid.norm_mask.assign(4, 1U); + problem.grid.omega_window.assign(4, 1.0); + problem.grid.omega = {1.0, 2.0, 3.0, 4.0}; + problem.grid.time_window.assign(4, 1.0); + problem.grid.carrier.assign(4, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(8, Complex{}); + problem.linear.prefactor_terms.assign(8, Complex(1.0, 0.0)); const std::vector initial{{1.0, 0.0}, {0.5, 0.25}, {0.0, 0.0}, {0.0, 0.0}, {0.25, 0.0}, {0.0, 0.5}, {0.0, 0.0}, {0.0, 0.0}}; constexpr auto flags = @@ -588,10 +596,10 @@ TEST(GeneralState, PackedStatisticsMatchStraightforwardHostReductions) { const double one = std::norm(std::complex( initial[mode * 4 + index].real(), initial[mode * 4 + index].imag())); power += one; - weighted += problem.omega[index] * one; + weighted += problem.grid.omega[index] * one; } EXPECT_NEAR(values[mode], weighted / power, 32.0 * kEpsilon); - EXPECT_NEAR(values[2 + mode], problem.dt / 4.0 * power, 32.0 * kEpsilon); + EXPECT_NEAR(values[2 + mode], problem.grid.dt / 4.0 * power, 32.0 * kEpsilon); } std::array, 2> time{ envelope_time(std::vector(initial.begin(), initial.begin() + 4), 4), @@ -625,11 +633,11 @@ TEST(GeneralState, EveryFieldFamilyMatchesTheStraightforwardHostResponse) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = general_problem(family); - std::vector initial(problem.n_omega); + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); constexpr double taper = 1.5; constexpr double dz = 0.02; @@ -641,7 +649,8 @@ TEST(GeneralState, EveryFieldFamilyMatchesTheStraightforwardHostResponse) { const auto result = state.field_to_host(); for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; - const double tolerance = 512.0 * kEpsilon * static_cast(problem.n_to) * + const double tolerance = 512.0 * kEpsilon * + static_cast(problem.dims.n_to) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -655,11 +664,11 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = raman_problem(family); - std::vector initial(problem.n_omega); + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); const auto derivative = raman_reference(problem, initial, rates); auto scalars = trial(problem); @@ -673,7 +682,7 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; const double tolerance = 512.0 * kEpsilon * - std::log2(2.0 * static_cast(problem.n_to)) * + std::log2(2.0 * static_cast(problem.dims.n_to)) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -682,10 +691,10 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { TEST(GeneralState, RamanMatchesDirectConvolutionForAShortFinalNodeChunk) { auto problem = raman_problem(native::FieldFamily::field_real); - problem.n_nodes = 3; - problem.node_chunk = 2; - problem.synthesis_matrix = {1.0, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.75}; + problem.dims.n_nodes = 3; + problem.dims.node_chunk = 2; + problem.transform.synthesis_matrix = {1.0, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.75}; const std::vector rates{0.125, 0.25}; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}}; const auto derivative = raman_reference(problem, initial, rates); @@ -700,7 +709,7 @@ TEST(GeneralState, RamanMatchesDirectConvolutionForAShortFinalNodeChunk) { for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; const double tolerance = 512.0 * kEpsilon * - std::log2(2.0 * static_cast(problem.n_to)) * + std::log2(2.0 * static_cast(problem.dims.n_to)) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -725,44 +734,44 @@ std::vector vector_raman_reference(const native::Problem& problem, const bool isotropic_only) { std::array, 2> modal; for (std::size_t pol = 0; pol < 2; ++pol) { - const auto offset = static_cast(pol * problem.n_omega); + const auto offset = static_cast(pol * problem.dims.n_omega); std::vector input(spectrum.begin() + offset, spectrum.begin() + offset + - static_cast(problem.n_omega)); - if (problem.family == native::FieldFamily::field_real) { - for (const double value : real_time(input, problem.n_to)) + static_cast(problem.dims.n_omega)); + if (problem.dims.family == native::FieldFamily::field_real) { + for (const double value : real_time(input, problem.dims.n_to)) modal[pol].emplace_back(value, 0.0); } else { - if (problem.family == native::FieldFamily::field_analytic) { + if (problem.dims.family == native::FieldFamily::field_analytic) { for (std::size_t j = 1; j < input.size(); ++j) input[j] *= 2.0; } - modal[pol] = envelope_time(input, problem.n_to); + modal[pol] = envelope_time(input, problem.dims.n_to); } } - std::array, 2> projected{std::vector(problem.n_to), - std::vector(problem.n_to)}; - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - const double synthesis = problem.synthesis_matrix[node]; - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::array, 2> projected{ + std::vector(problem.dims.n_to), std::vector(problem.dims.n_to)}; + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + const double synthesis = problem.transform.synthesis_matrix[node]; + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { std::array field{synthesis * modal[0][time], synthesis * modal[1][time]}; - if (problem.family != native::FieldFamily::envelope) + if (problem.dims.family != native::FieldFamily::envelope) for (auto& value : field) value = Complex(value.real(), 0.0); for (std::size_t previous = 0; previous <= time; ++previous) { const Complex x = synthesis * modal[0][previous]; const Complex y = synthesis * modal[1][previous]; - const double half = problem.raman_thg ? 1.0 : 0.5; + const double half = problem.raman.thg ? 1.0 : 0.5; const double xx = half * (x.real() * x.real() + x.imag() * x.imag()); const double yy = half * (y.real() * y.real() + y.imag() * y.imag()); const double xy = half * (x.real() * y.real() + x.imag() * y.imag()); const std::array, 2> q{{{xx, xy}, {xy, yy}}}; const auto lag = time - previous; - const double delay = static_cast(lag) * problem.dt_fine; - double hv = problem.raman_undamped[2 * problem.n_to + lag] * + const double delay = static_cast(lag) * problem.grid.dt_fine; + double hv = problem.raman.undamped[2 * problem.dims.n_to + lag] * std::exp(-rates[1] * delay); - double hr = problem.raman_undamped[lag] * std::exp(-rates[0] * delay); + double hr = problem.raman.undamped[lag] * std::exp(-rates[0] * delay); if (isotropic_only) { hv += hr; hr = 0.0; @@ -772,9 +781,10 @@ std::vector vector_raman_reference(const native::Problem& problem, for (std::size_t other = 0; other < 2; ++other) value += 0.5 * hr * (3.0 * q[pol][other] - (pol == other ? xx + yy : 0.0)) * field[other]; - projected[pol][time] += problem.projection_matrix[node] * - problem.raman_epsilon0 * problem.raman_fraction * - problem.dt_fine * problem.time_window[time] * value; + projected[pol][time] += problem.transform.projection_matrix[node] * + problem.raman.epsilon0 * problem.raman.fraction * + problem.grid.dt_fine * + problem.grid.time_window[time] * value; } } } @@ -784,7 +794,7 @@ std::vector vector_raman_reference(const native::Problem& problem, const auto transformed = dft(component); derivative.insert(derivative.end(), transformed.begin(), transformed.begin() + - static_cast(problem.n_omega)); + static_cast(problem.dims.n_omega)); } return derivative; } @@ -796,29 +806,30 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) for (const std::size_t count : {8U, 12U, 16U}) { for (const bool isotropic_only : {false, true}) { auto problem = raman_problem(family); - const auto old_count = problem.n_to; - const auto old_rows = problem.raman_undamped; - problem.n_to = count; - problem.time_window.assign(count, 1.0); - problem.carrier.assign(count, Complex(1.0, 0.0)); - problem.raman_undamped.assign(problem.n_raman_terms * 2 * count, 0.0); - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) + const auto old_count = problem.dims.n_to; + const auto old_rows = problem.raman.undamped; + problem.dims.n_to = count; + problem.grid.time_window.assign(count, 1.0); + problem.grid.carrier.assign(count, Complex(1.0, 0.0)); + problem.raman.undamped.assign(problem.raman.n_terms * 2 * count, 0.0); + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) for (std::size_t sample = 0; sample < old_count; ++sample) - problem.raman_undamped[term * 2 * count + sample] = + problem.raman.undamped[term * 2 * count + sample] = old_rows[term * 2 * old_count + sample]; - problem.n_pol_state = 2; - problem.raman_tensor_capable = true; - problem.raman_tensor_weights = isotropic_only + problem.dims.n_pol_state = 2; + problem.raman.tensor_capable = true; + problem.raman.tensor_weights = isotropic_only ? std::vector{1.0, 0.0, 1.0, 0.0} : std::vector{0.25, 0.75, 1.0, 0.0}; - problem.n_nodes = 3; - problem.node_chunk = 2; - problem.synthesis_matrix = {1.0, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.75}; - problem.phi_terms.assign(2 * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(2 * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_nodes = 3; + problem.dims.node_chunk = 2; + problem.transform.synthesis_matrix = {1.0, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.75}; + problem.linear.phi_terms.assign(2 * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(2 * problem.dims.n_omega, + Complex(1.0, 0.0)); const std::vector rates{0.125, 0.25}; - std::vector initial(2 * problem.n_omega); + std::vector initial(2 * problem.dims.n_omega); for (std::size_t j = 0; j < initial.size(); ++j) initial[j] = Complex(std::cos(static_cast(j)), 0.25 * std::sin(0.7 * static_cast(j))); @@ -829,8 +840,8 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) // cancellation in an individual spectral sample; real and imaginary input // components <= 1. const double tolerance = - (3.0 * 512.0 * std::log2(2.0 * static_cast(problem.n_to)) + - 64.0 * static_cast(problem.n_to * problem.n_nodes)) * + (3.0 * 512.0 * std::log2(2.0 * static_cast(problem.dims.n_to)) + + 64.0 * static_cast(problem.dims.n_to * problem.dims.n_nodes)) * kEpsilon; ASSERT_GT(tolerance, 0.0); native::GeneralState state(problem); @@ -850,11 +861,11 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) for (std::size_t j = 0; j < actual.size(); ++j) expect_complex_near(repeated[j], actual[j], tolerance); const auto ledger = state.memory_ledger(); - EXPECT_EQ(ledger.at("node_raman_drive"), problem.node_chunk * - problem.raman_drives() * 2 * - problem.n_to * sizeof(double)); + EXPECT_EQ(ledger.at("node_raman_drive"), + problem.dims.node_chunk * problem.raman_drives() * 2 * + problem.dims.n_to * sizeof(double)); EXPECT_EQ(ledger.at("raman_tensor_weights"), - problem.n_raman_terms * 2 * sizeof(double)); + problem.raman.n_terms * 2 * sizeof(double)); } } } @@ -865,7 +876,7 @@ TEST(GeneralState, RamanTensorWeightsAreValidatedBeforeAllocation) { {std::vector{1.0}, std::vector{1.0, 0.5, 1.0, 0.0}, std::vector{1.0, 0.0, std::numeric_limits::quiet_NaN(), 0.0}}) { auto problem = raman_problem(native::FieldFamily::envelope); - problem.raman_tensor_weights = weights; + problem.raman.tensor_weights = weights; EXPECT_THROW(static_cast(native::GeneralState(std::move(problem))), std::invalid_argument); } @@ -873,20 +884,20 @@ TEST(GeneralState, RamanTensorWeightsAreValidatedBeforeAllocation) { TEST(GeneralState, RamanValidationPrecedesFieldAllocation) { auto vector = raman_problem(native::FieldFamily::field_real); - vector.n_pol_state = 2; - vector.phi_terms.assign(2 * vector.n_omega, Complex{}); - vector.prefactor_terms.assign(2 * vector.n_omega, Complex(1.0, 0.0)); + vector.dims.n_pol_state = 2; + vector.linear.phi_terms.assign(2 * vector.dims.n_omega, Complex{}); + vector.linear.prefactor_terms.assign(2 * vector.dims.n_omega, Complex(1.0, 0.0)); EXPECT_THROW(static_cast(native::GeneralState(std::move(vector))), std::invalid_argument); auto wrong = raman_problem(native::FieldFamily::field_real); - wrong.raman_undamped.pop_back(); + wrong.raman.undamped.pop_back(); EXPECT_THROW(static_cast(native::GeneralState(std::move(wrong))), std::invalid_argument); auto valid = raman_problem(native::FieldFamily::field_real); native::GeneralState state(valid); - state.set_initial(std::vector(valid.n_omega), {}); + state.set_initial(std::vector(valid.dims.n_omega), {}); auto rates = trial(valid); rates.raman_rates = {-1.0, 0.0}; EXPECT_THROW(static_cast(state.trial_step(0.01, rates)), std::invalid_argument); @@ -894,26 +905,27 @@ TEST(GeneralState, RamanValidationPrecedesFieldAllocation) { TEST(GeneralState, TwoStatePolarisationsUseTheSharedVectorModulus) { auto problem = general_problem(native::FieldFamily::field_real); - problem.n_pol_state = 2; - problem.phi_terms.assign(2 * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(2 * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_pol_state = 2; + problem.linear.phi_terms.assign(2 * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(2 * problem.dims.n_omega, Complex(1.0, 0.0)); const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; std::array, 2> time{ real_time(std::vector(initial.begin(), initial.begin() + 3), - problem.n_to), + problem.dims.n_to), real_time(std::vector(initial.begin() + 3, initial.end()), - problem.n_to)}; + problem.dims.n_to)}; std::array, 2> derivative; for (std::size_t pol = 0; pol < 2; ++pol) { - std::vector response(problem.n_to); - for (std::size_t sample = 0; sample < problem.n_to; ++sample) { - const double first = problem.synthesis * time[0][sample]; - const double second = problem.synthesis * time[1][sample]; + std::vector response(problem.dims.n_to); + for (std::size_t sample = 0; sample < problem.dims.n_to; ++sample) { + const double first = problem.transform.synthesis * time[0][sample]; + const double second = problem.transform.synthesis * time[1][sample]; const double modulus = first * first + second * second; - response[sample] = Complex(problem.projection * problem.kerr_coefficient * - modulus * (pol == 0 ? first : second), - 0.0); + response[sample] = + Complex(problem.transform.projection * problem.kerr.coefficient * modulus * + (pol == 0 ? first : second), + 0.0); } const auto transformed = dft(response); derivative[pol] = {transformed.begin(), transformed.begin() + 3}; @@ -925,51 +937,59 @@ TEST(GeneralState, TwoStatePolarisationsUseTheSharedVectorModulus) { state.finish(dz); const auto result = state.field_to_host(); for (std::size_t pol = 0; pol < 2; ++pol) { - for (std::size_t index = 0; index < problem.n_omega; ++index) { - const auto flat = pol * problem.n_omega + index; + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + const auto flat = pol * problem.dims.n_omega + index; const Complex expected = initial[flat] + dz * derivative[pol][index]; expect_complex_near(result[flat], expected, - 1024.0 * static_cast(problem.n_to) * kEpsilon); + 1024.0 * static_cast(problem.dims.n_to) * kEpsilon); } } } TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { auto problem = plasma_problem(); - problem.stash_plasma = false; + problem.plasma.stash = false; const double theta = - 2.0 * std::numbers::pi_v / static_cast(problem.n_to); - problem.omega = {0.0, theta / problem.dt_fine, 2.0 * theta / problem.dt_fine}; - problem.active_mask[0] = 0; - problem.field_min = 1e6; - problem.field_max = 1e7; - problem.log_field_min = std::log(problem.field_min); - problem.d_log_field = std::log(10.0); + 2.0 * std::numbers::pi_v / static_cast(problem.dims.n_to); + problem.grid.omega = {0.0, theta / problem.grid.dt_fine, + 2.0 * theta / problem.grid.dt_fine}; + problem.grid.active_mask[0] = 0; + problem.plasma.field_min = 1e6; + problem.plasma.field_max = 1e7; + problem.plasma.log_field_min = std::log(problem.plasma.field_min); + problem.plasma.d_log_field = std::numbers::ln10; constexpr double population = 0.25; // The unit mode at bin one produces E_n = 2*synthesis/N*cos(n theta). // The exact finite trapezoid sum of cos is dt*cot(theta/2)*sin(n theta)/2. - const double amplitude = 2.0 * problem.synthesis / static_cast(problem.n_to); - const double current_scale = population * problem.charge_mass_ratio * amplitude * - problem.dt_fine / (2.0 * std::tan(theta / 2.0)); - std::vector product(problem.n_to), edge(problem.n_to); + const double amplitude = + 2.0 * problem.transform.synthesis / static_cast(problem.dims.n_to); + const double current_scale = population * problem.plasma.charge_mass_ratio * + amplitude * problem.grid.dt_fine / + (2.0 * std::tan(theta / 2.0)); + std::vector product(problem.dims.n_to); + std::vector edge(problem.dims.n_to); double operand = 0.0; - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { const double angle = theta * static_cast(time); - problem.time_window[time] = 0.5 * (1.0 - std::cos(angle)); - problem.time_window_derivative[time] = - 0.5 * theta / problem.dt_fine * std::sin(angle); + problem.grid.time_window[time] = 0.5 * (1.0 - std::cos(angle)); + problem.grid.time_window_derivative[time] = + 0.5 * theta / problem.grid.dt_fine * std::sin(angle); const double current = current_scale * std::sin(angle); - product[time] = problem.projection * problem.time_window[time] * current; - edge[time] = problem.projection * problem.time_window_derivative[time] * current; - operand += Kokkos::abs(product[time]) / problem.omega[1] + - Kokkos::abs(edge[time]) / (problem.omega[1] * problem.omega[1]); + product[time] = + problem.transform.projection * problem.grid.time_window[time] * current; + edge[time] = problem.transform.projection * + problem.grid.time_window_derivative[time] * current; + operand += + Kokkos::abs(product[time]) / problem.grid.omega[1] + + Kokkos::abs(edge[time]) / (problem.grid.omega[1] * problem.grid.omega[1]); } // Independent trigonometric sums, N trapezoids and three transforms. The bound // uses both absolute operands before their cancellation, fixed before execution. const double tolerance = - 512.0 * static_cast(problem.n_to) * kEpsilon * operand; + 512.0 * static_cast(problem.dims.n_to) * kEpsilon * operand; ASSERT_GT(tolerance, 0.0); - const auto x = dft(product), z = dft(edge); + const auto x = dft(product); + const auto z = dft(edge); native::GeneralState state(problem); auto scalars = trial(problem); scalars.preionisation = {population}; @@ -977,13 +997,13 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { static_cast(state.trial_step(0.0, scalars)); const auto actual = state.stage_to_host(0); EXPECT_EQ(actual[0], Complex{}); - for (std::size_t bin = 1; bin < problem.n_omega; ++bin) { - const double omega = problem.omega[bin]; + for (std::size_t bin = 1; bin < problem.dims.n_omega; ++bin) { + const double omega = problem.grid.omega[bin]; const Complex expected = Complex(0.0, -1.0 / omega) * x[bin] + z[bin] / (omega * omega); expect_complex_near(actual[bin], expected, tolerance); } - state.set_initial(std::vector(problem.n_omega), {}); + state.set_initial(std::vector(problem.dims.n_omega), {}); static_cast(state.trial_step(0.0, scalars)); for (const auto value : state.stage_to_host(0)) EXPECT_EQ(value, Complex{}); @@ -991,15 +1011,15 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { TEST(GeneralState, HeldCurrentRefusesMissingDerivativeAndNonpositiveActiveFrequency) { auto problem = plasma_problem(); - problem.time_window_derivative.clear(); + problem.grid.time_window_derivative.clear(); EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); problem = plasma_problem(); - problem.time_window_derivative[0] = std::numeric_limits::infinity(); + problem.grid.time_window_derivative[0] = std::numeric_limits::infinity(); EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); problem = plasma_problem(); - problem.omega[0] = 0.0; + problem.grid.omega[0] = 0.0; EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); - problem.active_mask[0] = 0; + problem.grid.active_mask[0] = 0; EXPECT_NO_THROW(native::GeneralState state(problem)); } @@ -1011,8 +1031,8 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { auto rejected = trial(problem); rejected.preionisation = {0.7}; constexpr double density = 4.0; - const double exposure = - problem.rate_max * problem.dt_fine * static_cast(problem.n_to - 1); + const double exposure = problem.plasma.rate_max * problem.grid.dt_fine * + static_cast(problem.dims.n_to - 1); const auto final_fraction = [exposure](const double preionisation) { return 1.0 - (1.0 - preionisation) * std::exp(-exposure); }; @@ -1027,7 +1047,7 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { const auto retained = state.statistics(flags, {}, {}, density, 1.0, 0.7, 1.0); ASSERT_EQ(retained.size(), 2); EXPECT_NEAR(retained[0], density * final_fraction(0.1), 64.0 * kEpsilon); - EXPECT_NEAR(retained[1], problem.rate_max, 32.0 * kEpsilon); + EXPECT_NEAR(retained[1], problem.plasma.rate_max, 32.0 * kEpsilon); state.clear_fsal(); const auto rescanned = state.statistics(flags, {}, {}, density, 1.0, 0.7, 1.0); EXPECT_NEAR(rescanned[0], density * final_fraction(0.7), 64.0 * kEpsilon); @@ -1038,23 +1058,24 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { auto problem = plasma_problem(); - problem.n_channels = 2; + problem.plasma.n_channels = 2; const std::array rates{0.125, 0.25}; for (std::size_t channel = 0; channel < 2; ++channel) { - const std::array parameters{problem.log_field_min, - problem.d_log_field, - problem.field_min, - problem.field_max, + const std::array parameters{problem.plasma.log_field_min, + problem.plasma.d_log_field, + problem.plasma.field_min, + problem.plasma.field_max, rates[channel], 1.0, 4.0 * static_cast(channel), - problem.charge_mass_ratio, - problem.ionisation_potential}; - problem.channel_parameters.insert(problem.channel_parameters.end(), - parameters.begin(), parameters.end()); + problem.plasma.charge_mass_ratio, + problem.plasma.ionisation_potential}; + problem.plasma.channel_parameters.insert(problem.plasma.channel_parameters.end(), + parameters.begin(), parameters.end()); const std::array coefficients{0.0, 0.0, 0.0, std::log(rates[channel])}; - problem.channel_coefficients.insert(problem.channel_coefficients.end(), - coefficients.begin(), coefficients.end()); + problem.plasma.channel_coefficients.insert( + problem.plasma.channel_coefficients.end(), coefficients.begin(), + coefficients.end()); } auto accepted = trial(problem); accepted.preionisation = {0.0}; @@ -1066,8 +1087,8 @@ TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { // operations enter each fraction, including exp, before statistics are read. constexpr double tolerance = 64.0 * kEpsilon; const auto expected = [&](const double initial, const std::size_t channel) { - return 1.0 - (1.0 - initial) * std::exp(-rates[channel] * problem.dt_fine * - static_cast(problem.n_to - 1)); + return 1.0 - (1.0 - initial) * std::exp(-rates[channel] * problem.grid.dt_fine * + static_cast(problem.dims.n_to - 1)); }; native::GeneralState state(problem); state.set_initial(std::vector{{2.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}}, {}); @@ -1094,65 +1115,66 @@ TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { TEST(GeneralState, MemoryLedgerMatchesConditionalAllocationFormulae) { auto problem = multimode_problem(); - problem.weighted_controller = true; - problem.plasma = true; - problem.stash_plasma = true; - problem.dt_fine = 0.125; - problem.charge_mass_ratio = 2.0; - problem.ionisation_potential = 3.0; - problem.log_field_min = std::log(0.01); - problem.d_log_field = std::log(1000.0); - problem.field_min = 0.01; - problem.field_max = 10.0; - problem.rate_max = 0.5; - problem.rate_intervals = 1; - problem.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; + problem.stepper.weighted_controller = true; + problem.plasma.enabled = true; + problem.plasma.stash = true; + problem.grid.dt_fine = 0.125; + problem.plasma.charge_mass_ratio = 2.0; + problem.plasma.ionisation_potential = 3.0; + problem.plasma.log_field_min = std::log(0.01); + problem.plasma.d_log_field = std::log(1000.0); + problem.plasma.field_min = 0.01; + problem.plasma.field_max = 10.0; + problem.plasma.rate_max = 0.5; + problem.plasma.rate_intervals = 1; + problem.plasma.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; native::GeneralState state(problem); const auto ledger = state.memory_ledger(); const std::size_t real = sizeof(double); const std::size_t complex = sizeof(Complex); - EXPECT_EQ(ledger.at("state"), problem.n_columns() * problem.n_omega * complex); - EXPECT_EQ(ledger.at("stages"), - problem.n_stages * problem.n_columns() * problem.n_omega * complex); + EXPECT_EQ(ledger.at("state"), problem.n_columns() * problem.dims.n_omega * complex); + EXPECT_EQ(ledger.at("stages"), problem.stepper.n_stages * problem.n_columns() * + problem.dims.n_omega * complex); EXPECT_EQ(ledger.at("transform_matrices"), - (problem.synthesis_matrix.size() + problem.projection_matrix.size() + - problem.projection_coarse_matrix.size()) * + (problem.transform.synthesis_matrix.size() + + problem.transform.projection_matrix.size() + + problem.transform.projection_coarse_matrix.size()) * real); EXPECT_EQ(ledger.at("stacked_projection"), - 2 * problem.n_columns() * problem.n_to * real); + 2 * problem.n_columns() * problem.dims.n_to * real); EXPECT_EQ(ledger.at("scaled_transform_matrices"), - 5 * problem.n_modes * problem.n_nodes * real); + 5 * problem.dims.n_modes * problem.dims.n_nodes * real); EXPECT_EQ(ledger.at("plasma_stash"), - 2 * problem.n_nodes * (1 + 2 * problem.n_channels) * real); + 2 * problem.dims.n_nodes * (1 + 2 * problem.plasma.n_channels) * real); // The device column's rows; on the host the band statistics' three rows per team // slot, at most one slot per thread (a trapezoid problem keeps no band rows). #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const auto chunk_rows = problem.node_chunk * problem.effective_node_pol(); + const auto chunk_rows = problem.dims.node_chunk * problem.effective_node_pol(); // With the trapezoid's binary64 tile totals, a row's tiles of 1,024 samples. - const auto tiles = (problem.n_to + 1023) / 1024; + const auto tiles = (problem.dims.n_to + 1023) / 1024; EXPECT_EQ(ledger.at("plasma_device_scan"), - (3 * problem.node_chunk * problem.n_to + 3 * chunk_rows * problem.n_to + - chunk_rows * tiles) * + (3 * problem.dims.node_chunk * problem.dims.n_to + + 3 * chunk_rows * problem.dims.n_to + chunk_rows * tiles) * real); #else // One slot per thread of the response operation's own width (problem-size policy). const auto response_elements = - problem.node_chunk * problem.effective_node_pol() * problem.n_to; + problem.dims.node_chunk * problem.effective_node_pol() * problem.dims.n_to; const int width = native::cpu_size_policy_enabled ? native::operation_threads( native::thread_policy(), native::test::table(), native::CpuPool::kernels, native::CpuOperation::response, - response_elements, problem.n_to) + response_elements, problem.dims.n_to) : native::ExecutionSpace().concurrency(); const auto slots = - std::min(problem.node_chunk, static_cast(std::max(1, width))); - EXPECT_EQ(ledger.at("plasma_device_scan"), 3 * slots * problem.n_to * real); + std::min(problem.dims.node_chunk, static_cast(std::max(1, width))); + EXPECT_EQ(ledger.at("plasma_device_scan"), 3 * slots * problem.dims.n_to * real); #endif EXPECT_EQ(ledger.at("weighted_control"), - (problem.n_omega + problem.n_columns()) * real); + (problem.dims.n_omega + problem.n_columns()) * real); EXPECT_EQ(ledger.at("error_partials"), 3 * problem.n_columns() * real); EXPECT_EQ(ledger.at("statistics_spectrum"), - problem.n_columns() * problem.n_t * complex); + problem.n_columns() * problem.dims.n_t * complex); } TEST(GeneralState, RamanMemoryLedgerMatchesTheConditionalViewFormulae) { @@ -1162,18 +1184,18 @@ TEST(GeneralState, RamanMemoryLedgerMatchesTheConditionalViewFormulae) { const auto real = sizeof(double); const auto complex = sizeof(Complex); EXPECT_EQ(ledger.at("node_raman_drive"), - problem.node_chunk * 2 * problem.n_to * real); + problem.dims.node_chunk * 2 * problem.dims.n_to * real); EXPECT_EQ(ledger.at("node_raman_spectrum"), - problem.node_chunk * (problem.n_to + 1) * complex); + problem.dims.node_chunk * (problem.dims.n_to + 1) * complex); EXPECT_EQ(ledger.at("node_raman_convolved"), - problem.node_chunk * 2 * problem.n_to * real); + problem.dims.node_chunk * 2 * problem.dims.n_to * real); EXPECT_EQ(ledger.at("raman_undamped"), - problem.n_raman_terms * 2 * problem.n_to * real); - EXPECT_EQ(ledger.at("raman_kernel_time"), 2 * problem.n_to * real); - EXPECT_EQ(ledger.at("raman_kernel_spectrum"), (problem.n_to + 1) * complex); + problem.raman.n_terms * 2 * problem.dims.n_to * real); + EXPECT_EQ(ledger.at("raman_kernel_time"), 2 * problem.dims.n_to * real); + EXPECT_EQ(ledger.at("raman_kernel_spectrum"), (problem.dims.n_to + 1) * complex); // Two slots of the packed stage scalars, the trial's step and its sample. EXPECT_EQ(ledger.at("stage_raman_rates"), - 2 * problem.n_stages * problem.n_raman_terms * real); + 2 * problem.stepper.n_stages * problem.raman.n_terms * real); native::GeneralState ordinary(multimode_problem()); const auto absent = ordinary.memory_ledger(); @@ -1374,7 +1396,6 @@ TEST(Gemm, EngineSelectsTheNativeOrderKernelWithoutAVendorBlas) { TEST(GeneralState, ProviderPolicyAndLifecycleAreIntrospectable) { native::GeneralState state(multimode_problem()); - const auto compiled = native::compiled_gemm_provider(); // A host build's hybrid policy sends every eligible transform to the vendor GEMM; a // device build keeps its shape guard, which this two-node problem is below. #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) @@ -1382,11 +1403,9 @@ TEST(GeneralState, ProviderPolicyAndLifecycleAreIntrospectable) { #else constexpr bool hybrid_vendor = true; #endif - const auto expected = - compiled == native::GemmProvider::custom || - (compiled == native::GemmProvider::hybrid && !hybrid_vendor) - ? std::string("custom") - : std::string(native::selected_blas_provider_name()); + const auto expected = hybrid_vendor + ? std::string(native::selected_blas_provider_name()) + : std::string("custom"); EXPECT_EQ(state.synthesis_provider(), expected); EXPECT_EQ(state.projection_provider(), expected); EXPECT_NE(state.gemm_shape_policy().find("every whole transform on the host"), @@ -1410,6 +1429,10 @@ struct SynthesisCapture { [](const char* name) { synthesis_regions.emplace_back(name); }); } ~SynthesisCapture() { Kokkos::Tools::Experimental::set_callbacks(previous); } + SynthesisCapture(const SynthesisCapture&) = delete; + SynthesisCapture& operator=(const SynthesisCapture&) = delete; + SynthesisCapture(SynthesisCapture&&) = delete; + SynthesisCapture& operator=(SynthesisCapture&&) = delete; }; // Five nodes and modes in chunks of three, the tail two. The square map takes the @@ -1419,35 +1442,39 @@ struct SynthesisCapture { // with the whole transform. void check_radial_analytic_chunks(const bool absorber, const char* expected_region) { auto problem = general_problem(native::FieldFamily::field_analytic); - problem.n_modes = 5; - problem.n_nodes = 5; - problem.node_chunk = 3; - problem.spatial_spectral_weights.assign(problem.n_modes, 1.0); - problem.spatial_on_axis.assign(problem.n_modes, 1.0); - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); - problem.in_stage_absorber = absorber; + problem.dims.n_modes = 5; + problem.dims.n_nodes = 5; + problem.dims.node_chunk = 3; + problem.spatial.spectral_weights.assign(problem.dims.n_modes, 1.0); + problem.spatial.on_axis.assign(problem.dims.n_modes, 1.0); + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.absorber.in_stage = absorber; if (absorber) { - problem.absorber_fine.resize(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) - problem.absorber_fine[time] = 0.01 * static_cast(time % 3); - } - problem.synthesis_matrix.resize(problem.n_nodes * problem.n_modes); - problem.projection_matrix.resize(problem.n_modes * problem.n_nodes); - for (std::size_t row = 0; row < problem.n_nodes; ++row) { - for (std::size_t column = 0; column < problem.n_modes; ++column) { - const auto index = row * problem.n_modes + column; - problem.synthesis_matrix[index] = + problem.absorber.fine.resize(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) + problem.absorber.fine[time] = 0.01 * static_cast(time % 3); + } + problem.transform.synthesis_matrix.resize(problem.dims.n_nodes * + problem.dims.n_modes); + problem.transform.projection_matrix.resize(problem.dims.n_modes * + problem.dims.n_nodes); + for (std::size_t row = 0; row < problem.dims.n_nodes; ++row) { + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) { + const auto index = row * problem.dims.n_modes + column; + problem.transform.synthesis_matrix[index] = row == column ? 1.0 : 0.125 * static_cast(row + 1); - problem.projection_matrix[index] = + problem.transform.projection_matrix[index] = row == column ? 0.75 : -0.0625 * static_cast(column + 1); } } - std::vector initial(problem.n_modes * problem.n_omega); + std::vector initial(problem.dims.n_modes * problem.dims.n_omega); for (std::size_t index = 0; index < initial.size(); ++index) initial[index] = Complex(0.01 * static_cast(index + 1), -0.02); auto whole_problem = problem; - whole_problem.node_chunk = whole_problem.n_nodes; + whole_problem.dims.node_chunk = whole_problem.dims.n_nodes; native::GeneralState whole(whole_problem); whole.set_initial(initial, {}); static_cast(whole.trial_step(0.01, trial(whole_problem))); @@ -1464,20 +1491,18 @@ void check_radial_analytic_chunks(const bool absorber, const char* expected_regi std::count(synthesis_regions.begin(), synthesis_regions.end(), expected_region); const auto custom = std::count(synthesis_regions.begin(), synthesis_regions.end(), "solprop_native.transform.synthesis.custom"); - const bool uses_blas = - native::compiled_gemm_provider() != native::GemmProvider::custom; - EXPECT_EQ(selected, uses_blas ? 2 : 0); - EXPECT_EQ(custom, uses_blas ? 0 : 2); + EXPECT_EQ(selected, 2); + EXPECT_EQ(custom, 0); // Only the absorber-free map keeps the retained spectra. - EXPECT_EQ(ledger.at("retained_synthesis") > 0, uses_blas && !absorber); + EXPECT_EQ(ledger.at("retained_synthesis") > 0, !absorber); chunked.finish(0.01); const auto actual = chunked.field_to_host(); // Both computations have one inverse/forward FFT and two length-five products; // carry their rounding through the cubic and the stage's scaled increment. const double tolerance = - 512.0 * kEpsilon * static_cast(problem.n_to) + - 4.0 * (static_cast(problem.n_modes) * kEpsilon / - (1.0 - static_cast(problem.n_modes) * kEpsilon)); + 512.0 * kEpsilon * static_cast(problem.dims.n_to) + + 4.0 * (static_cast(problem.dims.n_modes) * kEpsilon / + (1.0 - static_cast(problem.dims.n_modes) * kEpsilon)); ASSERT_GT(tolerance, 0.0); for (std::size_t index = 0; index < actual.size(); ++index) expect_complex_near(actual[index], expected[index], @@ -1497,100 +1522,104 @@ TEST(GeneralState, SquareRadialAnalyticChunksSynthesiseRetainedSpectra) { TEST(GeneralState, ChunkedBlasBuffersHaveExactPersistentByteCost) { auto problem = multimode_problem(3); - problem.n_to = 1024; - problem.n_nodes = 8; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.synthesis_matrix.assign(problem.n_nodes * problem.n_modes, 0.5); - problem.projection_matrix.assign(problem.n_modes * problem.n_nodes, 0.25); - problem.projection_coarse_matrix.assign(problem.n_modes * problem.n_nodes, 0.125); + problem.dims.n_to = 1024; + problem.dims.n_nodes = 8; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes * problem.dims.n_modes, + 0.5); + problem.transform.projection_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.25); + problem.transform.projection_coarse_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.125); native::GeneralState state(problem); const auto ledger = state.memory_ledger(); - const auto provider = native::compiled_gemm_provider(); - const auto expected = provider == native::GemmProvider::custom - ? 0U - : (3U + 2U) * problem.n_modes * 3U * sizeof(double); - EXPECT_EQ(ledger.at("chunk_transform_buffers"), expected); - - if (provider != native::GemmProvider::custom) { - auto whole_problem = problem; - whole_problem.node_chunk = whole_problem.n_nodes; - const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, - {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; - native::GeneralState whole(whole_problem); - whole.set_initial(initial, {}); - static_cast(whole.trial_step(0.01, trial(whole_problem))); - whole.finish(0.01); - state.set_initial(initial, {}); - static_cast(state.trial_step(0.01, trial(problem))); - state.finish(0.01); - const auto reference = whole.field_to_host(); - const auto chunked = state.field_to_host(); - const auto dot_length = static_cast(problem.n_nodes); - const double dot_gamma = dot_length * kEpsilon / (1.0 - dot_length * kEpsilon); - const double tolerance = - 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.n_to))) + - 4.0 * dot_gamma; - for (std::size_t index = 0; index < chunked.size(); ++index) - expect_complex_near(chunked[index], reference[index], - tolerance * std::max(1.0, Kokkos::abs(reference[index]))); - } + EXPECT_EQ(ledger.at("chunk_transform_buffers"), + (3U + 2U) * problem.dims.n_modes * 3U * sizeof(double)); + + auto whole_problem = problem; + whole_problem.dims.node_chunk = whole_problem.dims.n_nodes; + const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, + {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; + native::GeneralState whole(whole_problem); + whole.set_initial(initial, {}); + static_cast(whole.trial_step(0.01, trial(whole_problem))); + whole.finish(0.01); + state.set_initial(initial, {}); + static_cast(state.trial_step(0.01, trial(problem))); + state.finish(0.01); + const auto reference = whole.field_to_host(); + const auto chunked = state.field_to_host(); + const auto dot_length = static_cast(problem.dims.n_nodes); + const double dot_gamma = dot_length * kEpsilon / (1.0 - dot_length * kEpsilon); + const double tolerance = + 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.dims.n_to))) + + 4.0 * dot_gamma; + for (std::size_t index = 0; index < chunked.size(); ++index) + expect_complex_near(chunked[index], reference[index], + tolerance * std::max(1.0, Kokkos::abs(reference[index]))); } -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) +#ifdef SOLPROP_NATIVE_TPL_BLAS_OPENBLAS TEST(GeneralState, OpenBlasEnvelopePolicyAllocatesAndExecutesComplexTransforms) { auto problem = multimode_problem(8); - problem.family = native::FieldFamily::envelope; - problem.n_t = 1024; - problem.n_to = 1024; - problem.n_omega = 1024; - problem.n_nodes = 8; - problem.crop_index.clear(); - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.crop_index.push_back(index); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); - problem.omega.assign(problem.n_omega, 1.0); - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); - problem.synthesis_matrix.assign(problem.n_nodes * problem.n_modes, 0.5); - problem.projection_matrix.assign(problem.n_modes * problem.n_nodes, 0.25); - problem.projection_coarse_matrix.assign(problem.n_modes * problem.n_nodes, 0.125); + problem.dims.family = native::FieldFamily::envelope; + problem.dims.n_t = 1024; + problem.dims.n_to = 1024; + problem.dims.n_omega = 1024; + problem.dims.n_nodes = 8; + problem.grid.crop_index.clear(); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.grid.crop_index.push_back(index); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); + problem.grid.omega.assign(problem.dims.n_omega, 1.0); + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes * problem.dims.n_modes, + 0.5); + problem.transform.projection_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.25); + problem.transform.projection_coarse_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.125); native::GeneralState state(problem); // Scalar envelopes use a real GEMM over the complex samples' interleaved view. EXPECT_EQ(state.synthesis_provider(), "kokkosblas-tpl-interleaved-real"); EXPECT_EQ(state.projection_provider(), "kokkosblas-tpl-interleaved-real"); const auto ledger = state.memory_ledger(); - const auto real_matrix_values = 3 * problem.n_modes * problem.n_nodes; - const auto complex_matrix_values = 5 * problem.n_modes * problem.n_nodes; + const auto real_matrix_values = 3 * problem.dims.n_modes * problem.dims.n_nodes; + const auto complex_matrix_values = 5 * problem.dims.n_modes * problem.dims.n_nodes; EXPECT_EQ(ledger.at("scaled_transform_matrices"), real_matrix_values * sizeof(double) + complex_matrix_values * sizeof(Complex)); EXPECT_EQ(ledger.at("stacked_projection"), - 2 * problem.n_modes * problem.n_to * sizeof(Complex)); + 2 * problem.dims.n_modes * problem.dims.n_to * sizeof(Complex)); - state.set_initial(std::vector(problem.n_modes * problem.n_omega), {}); + state.set_initial(std::vector(problem.dims.n_modes * problem.dims.n_omega), + {}); const auto partials = state.trial_step(0.01, trial(problem)); // Exact: a zero field has zero norms and a zero error. - EXPECT_EQ(partials, std::vector(3 * problem.n_modes, 0.0)); + EXPECT_EQ(partials, std::vector(3 * problem.dims.n_modes, 0.0)); } #endif TEST(GeneralState, RejectsOverflowingExtentsBeforeAllocation) { auto column_overflow = general_problem(native::FieldFamily::field_real); - column_overflow.n_modes = std::numeric_limits::max(); - column_overflow.n_pol_state = 2; + column_overflow.dims.n_modes = std::numeric_limits::max(); + column_overflow.dims.n_pol_state = 2; EXPECT_THROW(static_cast(native::GeneralState(std::move(column_overflow))), std::length_error); auto field_overflow = general_problem(native::FieldFamily::field_real); - field_overflow.n_modes = - std::numeric_limits::max() / field_overflow.n_omega + 1; + field_overflow.dims.n_modes = + std::numeric_limits::max() / field_overflow.dims.n_omega + 1; EXPECT_THROW(static_cast(native::GeneralState(std::move(field_overflow))), std::length_error); } @@ -1600,35 +1629,35 @@ TEST(GeneralState, RejectsOverflowingExtentsBeforeAllocation) { // (x, y) are ix + 3 iy, and each column's operator subtracts its channel's wall term. native::Problem planar_problem(const std::size_t rows_per_chunk) { auto problem = base_problem(native::FieldFamily::field_real); - problem.n_p = 2; - problem.n_x = 3; - problem.n_y = 4; - problem.n_modes = problem.n_p * problem.n_y; - problem.n_nodes = problem.n_x * problem.n_y; - problem.node_chunk = rows_per_chunk * problem.n_x; - problem.nonlinear = true; - problem.phi_terms.clear(); - problem.prefactor_terms.clear(); - problem.synthesis_matrix = {1.0, 0.5, -0.25, 1.5, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.125, 0.375, 0.625, -0.25}; - problem.spatial_k_squared = {{40.0, 0.5}, {60.0, 1.0}, {90.0, 1.5}}; - problem.spatial_k_squared_low = {{1e-15, 0.0}, {-2e-15, 0.0}, {0.0, 1e-16}}; - problem.spatial_numerator = {{0.0, -1.0}, {0.0, -2.0}, {0.0, -3.0}}; - problem.spatial_clamped = {{0.0, -0.25}, {0.0, -0.5}, {0.0, -0.75}}; - problem.spatial_frame = {5.0, 6.0, 7.0}; - problem.spatial_spectral_rate = {0.0, 0.25, 0.5}; - problem.spatial_user_rate = {0.0, 0.0, 0.0}; + problem.spatial.n_p = 2; + problem.spatial.n_x = 3; + problem.spatial.n_y = 4; + problem.dims.n_modes = problem.spatial.n_p * problem.spatial.n_y; + problem.dims.n_nodes = problem.spatial.n_x * problem.spatial.n_y; + problem.dims.node_chunk = rows_per_chunk * problem.spatial.n_x; + problem.kerr.nonlinear = true; + problem.linear.phi_terms.clear(); + problem.linear.prefactor_terms.clear(); + problem.transform.synthesis_matrix = {1.0, 0.5, -0.25, 1.5, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.125, 0.375, 0.625, -0.25}; + problem.spatial.k_squared = {{40.0, 0.5}, {60.0, 1.0}, {90.0, 1.5}}; + problem.spatial.k_squared_low = {{1e-15, 0.0}, {-2e-15, 0.0}, {0.0, 1e-16}}; + problem.spatial.numerator = {{0.0, -1.0}, {0.0, -2.0}, {0.0, -3.0}}; + problem.spatial.clamped = {{0.0, -0.25}, {0.0, -0.5}, {0.0, -0.75}}; + problem.spatial.frame = {5.0, 6.0, 7.0}; + problem.spatial.spectral_rate = {0.0, 0.25, 0.5}; + problem.spatial.user_rate = {0.0, 0.0, 0.0}; // k_y^2 per column, the channel fastest; the last row is evanescent in the first // bin, so both branches of the operator are exercised. const std::vector ky_squared{0.0, 4.0, 45.0, 4.0}; - for (std::size_t column = 0; column < problem.n_modes; ++column) { - problem.spatial_kperp_sq.push_back(ky_squared[column / problem.n_p]); - problem.spatial_kperp_sq_low.push_back(column == 3 ? 1e-16 : 0.0); - } - problem.spatial_absorber_rate.assign(problem.n_modes, 0.0); - problem.spatial_kperp_rate.assign(problem.n_modes, 0.0); - problem.spatial_evanescent_rate = 200.0; - problem.spatial_kappa_sq = {{1.0, 0.25}, {2.0, 0.5}, {3.0, 0.75}, + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) { + problem.spatial.kperp_sq.push_back(ky_squared[column / problem.spatial.n_p]); + problem.spatial.kperp_sq_low.push_back(column == 3 ? 1e-16 : 0.0); + } + problem.spatial.absorber_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.kperp_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.evanescent_rate = 200.0; + problem.spatial.kappa_sq = {{1.0, 0.25}, {2.0, 0.5}, {3.0, 0.75}, {9.0, 1.0}, {12.0, 1.5}, {15.0, 2.0}}; return problem; } @@ -1640,19 +1669,21 @@ struct PlanarElement { PlanarElement planar_element(const native::Problem& problem, const std::size_t column, const std::size_t omega) { const Complex square = - (problem.spatial_k_squared[omega] - problem.spatial_kperp_sq[column]) + - (problem.spatial_k_squared_low[omega] - problem.spatial_kperp_sq_low[column]) - - problem.spatial_kappa_sq[(column % problem.n_p) * problem.n_omega + omega]; + (problem.spatial.k_squared[omega] - problem.spatial.kperp_sq[column]) + + (problem.spatial.k_squared_low[omega] - problem.spatial.kperp_sq_low[column]) - + problem.spatial + .kappa_sq[(column % problem.spatial.n_p) * problem.dims.n_omega + omega]; const std::complex root = std::sqrt(std::complex(square.real(), square.imag())); const bool propagating = square.real() > 0.0; const double attenuation = propagating ? root.imag() - : std::min(std::abs(root.imag()), problem.spatial_evanescent_rate); - return {Complex(-attenuation, -(root.real() - problem.spatial_frame[omega])) - - 0.5 * problem.spatial_spectral_rate[omega], - propagating ? problem.spatial_numerator[omega] / root.real() - : problem.spatial_clamped[omega]}; + : std::min(std::abs(root.imag()), problem.spatial.evanescent_rate); + return {.linear = + Complex(-attenuation, -(root.real() - problem.spatial.frame[omega])) - + 0.5 * problem.spatial.spectral_rate[omega], + .prefactor = propagating ? problem.spatial.numerator[omega] / root.real() + : problem.spatial.clamped[omega]}; } // The stage straight from the definitions: the inverse DFT in y, each row's real time @@ -1660,8 +1691,10 @@ PlanarElement planar_element(const native::Problem& problem, const std::size_t c // and each column's prefactor. std::vector planar_reference(const native::Problem& problem, const std::vector& columns) { - const auto n_p = problem.n_p, n_x = problem.n_x, n_y = problem.n_y; - const auto n = problem.n_omega; + const auto n_p = problem.spatial.n_p; + const auto n_x = problem.spatial.n_x; + const auto n_y = problem.spatial.n_y; + const auto n = problem.dims.n_omega; std::vector> rows(n_p * n_y); for (std::size_t iy = 0; iy < n_y; ++iy) for (std::size_t p = 0; p < n_p; ++p) { @@ -1673,25 +1706,26 @@ std::vector planar_reference(const native::Problem& problem, cis(2.0 * std::numbers::pi_v * static_cast(ky * iy) / static_cast(n_y)) / static_cast(n_y); - rows[p + n_p * iy] = real_time(spectrum, problem.n_to); + rows[p + n_p * iy] = real_time(spectrum, problem.dims.n_to); } std::vector projected(n_p * n_y * n); for (std::size_t iy = 0; iy < n_y; ++iy) { - std::vector> polarisation(n_x, - std::vector(problem.n_to)); + std::vector> polarisation( + n_x, std::vector(problem.dims.n_to)); for (std::size_t ix = 0; ix < n_x; ++ix) - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double field = 0.0; for (std::size_t p = 0; p < n_p; ++p) - field += problem.synthesis_matrix[ix * n_p + p] * rows[p + n_p * iy][time]; - polarisation[ix][time] = problem.kerr_coefficient * field * field * field; + field += problem.transform.synthesis_matrix[ix * n_p + p] * + rows[p + n_p * iy][time]; + polarisation[ix][time] = problem.kerr.coefficient * field * field * field; } for (std::size_t p = 0; p < n_p; ++p) { - std::vector signal(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) + std::vector signal(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) for (std::size_t ix = 0; ix < n_x; ++ix) - signal[time] += - problem.projection_matrix[p * n_x + ix] * polarisation[ix][time]; + signal[time] += problem.transform.projection_matrix[p * n_x + ix] * + polarisation[ix][time]; const auto spectrum = dft(signal); for (std::size_t omega = 0; omega < n; ++omega) projected[(p + n_p * iy) * n + omega] = spectrum[omega]; @@ -1708,7 +1742,7 @@ std::vector planar_reference(const native::Problem& problem, static_cast(n_y)); const auto column = p + n_p * ky; result[column * n + omega] = planar_element(problem, column, omega).prefactor * - problem.omega_window[omega] * sum; + problem.grid.omega_window[omega] * sum; } return result; } @@ -1723,7 +1757,7 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { // One y row per chunk (a partial chunk nowhere), then three (a partial last one). for (const std::size_t rows_per_chunk : {std::size_t{1}, std::size_t{3}}) { const auto problem = planar_problem(rows_per_chunk); - ASSERT_EQ(initial.size(), problem.n_modes * problem.n_omega); + ASSERT_EQ(initial.size(), problem.dims.n_modes * problem.dims.n_omega); const auto derivative = planar_reference(problem, initial); constexpr double dz = 0.01; auto scalars = trial(problem); @@ -1741,9 +1775,9 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { // weighted update (one stage, unit weight), which carries the wall terms. state.finish(dz); const auto result = state.field_to_host(); - for (std::size_t column = 0; column < problem.n_modes; ++column) - for (std::size_t omega = 0; omega < problem.n_omega; ++omega) { - const auto index = column * problem.n_omega + omega; + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) + for (std::size_t omega = 0; omega < problem.dims.n_omega; ++omega) { + const auto index = column * problem.dims.n_omega + omega; const Complex exponent = dz * planar_element(problem, column, omega).linear; const Complex expected = std::exp(exponent.real()) * cis(exponent.imag()) * (initial[index] + dz * derivative[index]); @@ -1755,22 +1789,22 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { TEST(GeneralState, HybridPlanarProblemsAreValidatedBeforeAllocation) { auto partial_row = planar_problem(1); - partial_row.node_chunk = 4; + partial_row.dims.node_chunk = 4; EXPECT_THROW(native::GeneralState state(partial_row), std::invalid_argument); auto short_kappa = planar_problem(1); - short_kappa.spatial_kappa_sq.pop_back(); + short_kappa.spatial.kappa_sq.pop_back(); EXPECT_THROW(native::GeneralState state(short_kappa), std::invalid_argument); auto coarse = planar_problem(1); - coarse.projection_coarse_matrix = coarse.projection_matrix; + coarse.transform.projection_coarse_matrix = coarse.transform.projection_matrix; EXPECT_THROW(native::GeneralState state(coarse), std::invalid_argument); auto vector = planar_problem(1); - vector.n_pol_state = 2; + vector.dims.n_pol_state = 2; EXPECT_THROW(native::GeneralState state(vector), std::invalid_argument); auto columns = planar_problem(1); - columns.n_p = 3; + columns.spatial.n_p = 3; EXPECT_THROW(native::GeneralState state(columns), std::invalid_argument); auto modal = general_problem(native::FieldFamily::field_real); - modal.spatial_kappa_sq = {{1.0, 0.0}}; + modal.spatial.kappa_sq = {{1.0, 0.0}}; EXPECT_THROW(native::GeneralState state(modal), std::invalid_argument); } @@ -1848,7 +1882,7 @@ std::vector filter_tolerance(const std::vector& row, std::abs(row[static_cast(2 * last - index)]); return std::abs(row[static_cast(index)]); }; - const double operations = static_cast(2 * half + 3); + const auto operations = static_cast(2 * half + 3); const double gamma = operations * kEpsilon / (1.0 - operations * kEpsilon); std::vector tolerance(row.size()); for (std::ptrdiff_t index = 0; index < samples; ++index) { @@ -1890,7 +1924,8 @@ team_band_scan(const std::vector& values, const std::size_t rows, Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, filtered); const auto host_prefixes = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, prefixes); - std::vector filtered_values(rows * samples), prefix_values(rows * samples); + std::vector filtered_values(rows * samples); + std::vector prefix_values(rows * samples); for (std::size_t row = 0; row < rows; ++row) for (std::size_t sample = 0; sample < samples; ++sample) { filtered_values[row * samples + sample] = host_filtered(row, sample); @@ -2311,8 +2346,11 @@ WindowedScan windowed_band_scan(const std::vector& row, Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, windowed); const auto host_totals = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, totals); - WindowedScan result{std::vector(samples), std::vector(samples), - std::vector(samples), window, host_totals(0)}; + WindowedScan result{.filtered = std::vector(samples), + .whole = std::vector(samples), + .windowed = std::vector(samples), + .window = window, + .total = host_totals(0)}; for (std::size_t i = 0; i < samples; ++i) { result.filtered[i] = host_filtered(0, i); result.whole[i] = host_whole(0, i); @@ -2433,7 +2471,7 @@ TEST(PlasmaBandScan, FilteringPreservesShortRowsBlockTailsAndDynamicRange) { TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = true; + problem.kerr.nonlinear = true; const std::vector initial = { {1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.05, 0.025}}; native::GeneralState state(problem); @@ -2450,7 +2488,7 @@ TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { largest = std::max(largest, std::abs(value)); ASSERT_GT(largest, 0.0); const double tolerance = - 2.0 * 512.0 * kEpsilon * static_cast(problem.n_to) * largest; + 2.0 * 512.0 * kEpsilon * static_cast(problem.dims.n_to) * largest; for (std::size_t index = 0; index < first.size(); ++index) EXPECT_NEAR(second[index], first[index], tolerance) << index; // Exact: a trial never writes the accepted state. @@ -2459,16 +2497,16 @@ TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { TEST(GeneralState, RejectsDuplicateCropBins) { auto problem = general_problem(native::FieldFamily::envelope); - problem.crop_index[1] = problem.crop_index[0]; + problem.grid.crop_index[1] = problem.grid.crop_index[0]; EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); } TEST(GeneralState, SplitStepAbsorberDampsOnlyAcceptedState) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = false; - problem.split_step_absorber = true; - problem.absorber_coarse.assign(problem.n_t, 2.0); - const std::vector initial(problem.n_omega, {1.0, -0.25}); + problem.kerr.nonlinear = false; + problem.absorber.split_step = true; + problem.absorber.coarse.assign(problem.dims.n_t, 2.0); + const std::vector initial(problem.dims.n_omega, {1.0, -0.25}); constexpr double dz = 0.1; native::GeneralState state(problem); state.set_initial(initial, {}); @@ -2488,11 +2526,11 @@ TEST(GeneralState, SplitStepAbsorberDampsOnlyAcceptedState) { TEST(GeneralState, InStageAbsorberPreservesFirstSameAsLastEligibility) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = false; - problem.in_stage_absorber = true; - problem.first_same_as_last = true; - problem.absorber_fine.assign(problem.n_to, 2.0); - const std::vector initial(problem.n_omega, {1.0, 0.0}); + problem.kerr.nonlinear = false; + problem.absorber.in_stage = true; + problem.stepper.first_same_as_last = true; + problem.absorber.fine.assign(problem.dims.n_to, 2.0); + const std::vector initial(problem.dims.n_omega, {1.0, 0.0}); constexpr double dz = 0.1; native::GeneralState state(problem); state.set_initial(initial, {}); @@ -2513,17 +2551,17 @@ TEST(GeneralState, InStageAbsorberPreservesFirstSameAsLastEligibility) { // source (`multimode_reference`) and the exact half-step flows. native::Problem low_storage_problem(const std::size_t flows) { auto problem = multimode_problem(); - problem.n_stages = 3; - problem.tableau_matrix.assign(9, 0.0); - problem.tableau_nodes = {0.0, 0.5, 1.0}; - problem.solution_weights = {1.0, 0.0, 0.0}; - problem.error_weights = {0.25, 0.0, -0.25}; - problem.projection_coarse_matrix.clear(); - problem.low_storage = true; - problem.low_storage_flows = flows; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.phi_terms[mode * problem.n_omega + index] = + problem.stepper.n_stages = 3; + problem.stepper.tableau_matrix.assign(9, 0.0); + problem.stepper.tableau_nodes = {0.0, 0.5, 1.0}; + problem.stepper.solution_weights = {1.0, 0.0, 0.0}; + problem.stepper.error_weights = {0.25, 0.0, -0.25}; + problem.transform.projection_coarse_matrix.clear(); + problem.stepper.low_storage = true; + problem.stepper.low_storage_flows = flows; + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.linear.phi_terms[mode * problem.dims.n_omega + index] = Complex(-0.3 * static_cast(index), 2.0 + static_cast(mode)); return problem; } @@ -2543,7 +2581,7 @@ std::vector low_storage_reference(const native::Problem& problem, const auto size = state.size(); std::vector flow(size); for (std::size_t element = 0; element < size; ++element) { - const Complex exponent = 0.5 * dz * problem.phi_terms[element]; + const Complex exponent = 0.5 * dz * problem.linear.phi_terms[element]; flow[element] = std::exp(exponent.real()) * Complex(std::cos(exponent.imag()), std::sin(exponent.imag())); } @@ -2554,7 +2592,8 @@ std::vector low_storage_reference(const native::Problem& problem, result[element] = base[element] + weight * slope[element]; return result; }; - std::vector picture(size), g1(size); + std::vector picture(size); + std::vector g1(size); const auto f1 = multimode_reference(problem, state); for (std::size_t element = 0; element < size; ++element) { picture[element] = flow[element] * state[element]; @@ -2617,8 +2656,8 @@ TEST(GeneralState, LowStoragePassesMatchTheHostInteractionPictureStep) { TEST(GeneralState, LowStorageStepsAreRefusedOutsideTheirState) { auto runge_kutta = low_storage_problem(1); - runge_kutta.low_storage = false; - runge_kutta.low_storage_flows = 0; + runge_kutta.stepper.low_storage = false; + runge_kutta.stepper.low_storage_flows = 0; native::GeneralState state(runge_kutta); state.set_initial(std::vector(6), {}); EXPECT_THROW(static_cast(state.low_storage_step( @@ -2627,7 +2666,7 @@ TEST(GeneralState, LowStorageStepsAreRefusedOutsideTheirState) { auto three = low_storage_problem(3); EXPECT_THROW(native::GeneralState too_many(three), std::invalid_argument); auto elided = low_storage_problem(0); - elided.low_storage_stage_register = false; + elided.stepper.low_storage_stage_register = false; EXPECT_THROW(native::GeneralState modal(elided), std::invalid_argument); native::GeneralState march(low_storage_problem(0)); march.set_initial(std::vector(6), {}); @@ -2653,20 +2692,20 @@ std::string construction_refusal(const native::Problem& problem) { // with no slab channels. native::Problem low_storage_free_problem() { auto problem = planar_problem(1); - problem.n_p = 0; - problem.n_modes = problem.n_x * problem.n_y; - problem.n_nodes = problem.n_modes; - problem.synthesis_matrix.clear(); - problem.projection_matrix.clear(); - problem.spatial_kappa_sq.clear(); + problem.spatial.n_p = 0; + problem.dims.n_modes = problem.spatial.n_x * problem.spatial.n_y; + problem.dims.n_nodes = problem.dims.n_modes; + problem.transform.synthesis_matrix.clear(); + problem.transform.projection_matrix.clear(); + problem.spatial.kappa_sq.clear(); // Construction only: the unit transverse square is k_x = 1, k_y = 0. - problem.spatial_k_x.assign(problem.n_x, 1.0); - problem.spatial_k_y.assign(problem.n_y, 0.0); - problem.spatial_kperp_sq.assign(problem.n_modes, 1.0); - problem.spatial_kperp_sq_low.assign(problem.n_modes, 0.0); - problem.spatial_absorber_rate.assign(problem.n_modes, 0.0); - problem.spatial_kperp_rate.assign(problem.n_modes, 0.0); - problem.low_storage = true; + problem.spatial.k_x.assign(problem.spatial.n_x, 1.0); + problem.spatial.k_y.assign(problem.spatial.n_y, 0.0); + problem.spatial.kperp_sq.assign(problem.dims.n_modes, 1.0); + problem.spatial.kperp_sq_low.assign(problem.dims.n_modes, 0.0); + problem.spatial.absorber_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.kperp_rate.assign(problem.dims.n_modes, 0.0); + problem.stepper.low_storage = true; return problem; } @@ -2674,23 +2713,23 @@ TEST(GeneralState, LowStorageRefusesRegistersItsFoldsCannotRead) { // The exchange fold of an in-stage absorber re-forms its input from the stage // register, so it is never elided there. auto elided = low_storage_free_problem(); - elided.low_storage_stage_register = false; + elided.stepper.low_storage_stage_register = false; EXPECT_EQ(construction_refusal(elided), ""); - elided.in_stage_absorber = true; - elided.absorber_fine.assign(elided.n_to, 1.0); + elided.absorber.in_stage = true; + elided.absorber.fine.assign(elided.dims.n_to, 1.0); EXPECT_NE(construction_refusal(elided).find("beside an in-stage absorber"), std::string::npos); // A separable fold applies one flow volume, a constant operator's table. auto table = low_storage_free_problem(); - table.low_storage_flows = 1; + table.stepper.low_storage_flows = 1; EXPECT_EQ(construction_refusal(table), ""); auto pair = low_storage_free_problem(); - pair.low_storage_flows = 2; + pair.stepper.low_storage_flows = 2; EXPECT_NE(construction_refusal(pair).find("at most one flow volume"), std::string::npos); auto planar = planar_problem(1); - planar.low_storage = true; - planar.low_storage_flows = 2; + planar.stepper.low_storage = true; + planar.stepper.low_storage_flows = 2; EXPECT_NE(construction_refusal(planar).find("at most one flow volume"), std::string::npos); } @@ -2757,7 +2796,8 @@ TEST(CpuThreadPolicy, AutomaticBoundariesAreMonotoneAndCapped) { native::test::table()[native::CpuOperation::modal].threshold, native::test::table()[native::CpuOperation::adams].full}; for (const int cap : {1, 2, 3, 8, 72}) { - native::ThreadPolicy policy{cap, cap, cap, cap}; + native::ThreadPolicy policy{ + .kernels = cap, .fft = cap, .raman_fft = cap, .raman_tail_fft = cap}; for (std::size_t index = 0; index < operations.size(); ++index) { const auto width = [&](const std::size_t elements) { return native::operation_threads(policy, native::test::table(), @@ -2786,7 +2826,8 @@ TEST(CpuThreadPolicy, AutomaticBoundariesAreMonotoneAndCapped) { } TEST(CpuThreadPolicy, ShortCopiesDoNotInheritTheStageWidth) { - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; EXPECT_EQ(native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, native::CpuOperation::copy, 8192), @@ -2807,10 +2848,11 @@ TEST(CpuThreadPolicy, SmoothGridSizesKeepThePowerOfTwoWidths) { // stage of 8004 elements and coarse FFT batch of 32000 samples take the widths of // 8196 and 32768, rather than falling to one thread just under a power-of-two // boundary. macOS keeps the earlier boundaries, where the serial ones are faster. -#if defined(__APPLE__) +#ifdef __APPLE__ GTEST_SKIP() << "macOS keeps the 8192/32768 boundaries"; #endif - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; const auto width = [&](const native::CpuOperation operation, const std::size_t n) { return native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, operation, n); @@ -2823,11 +2865,15 @@ TEST(CpuThreadPolicy, SmoothGridSizesKeepThePowerOfTwoWidths) { } TEST(CpuThreadPolicy, CoupledMatrixWindowRetainsFullWidth) { - native::ThreadPolicy policy{72, 72, 72, 72, -1}; + native::ThreadPolicy policy{.kernels = 72, + .fft = 72, + .raman_fft = 72, + .raman_tail_fft = 72, + .band_tiles = -1}; // Active bins * modes^3 in the measured three-mode wall, independent of runtime. - EXPECT_EQ(native::operation_threads(policy, native::test::table(), - native::CpuPool::kernels, - native::CpuOperation::flow, 1433 * 3 * 3 * 3), + EXPECT_EQ(native::operation_threads( + policy, native::test::table(), native::CpuPool::kernels, + native::CpuOperation::flow, std::size_t{1433} * 3 * 3 * 3), 72); } @@ -2835,7 +2881,8 @@ TEST(CpuThreadPolicy, ResponseShapeThresholdsAreMonotoneAndCapped) { using native::CpuOperation; using native::CpuPool; for (const int cap : {1, 3, 18, 72}) { - native::ThreadPolicy policy{cap, cap, cap, cap}; + native::ThreadPolicy policy{ + .kernels = cap, .fft = cap, .raman_fft = cap, .raman_tail_fft = cap}; const auto width = [&](const std::size_t rows, const std::size_t samples) { return native::operation_threads(policy, native::test::table(), CpuPool::kernels, CpuOperation::response, rows * samples, samples); @@ -2871,7 +2918,8 @@ TEST(CpuThreadPolicy, ResponseShapeThresholdsAreMonotoneAndCapped) { TEST(CpuThreadPolicy, ExplicitPoolsBypassSizingIndependently) { using native::CpuOperation; using native::CpuPool; - native::ThreadPolicy policy{7, 6, 5, 4}; + native::ThreadPolicy policy{ + .kernels = 7, .fft = 6, .raman_fft = 5, .raman_tail_fft = 4}; const std::array pools{CpuPool::kernels, CpuPool::fft, CpuPool::raman_fft, CpuPool::raman_tail_fft}; for (unsigned requested = 0; requested < pools.size(); ++requested) { @@ -2899,7 +2947,8 @@ TEST(CpuThreadPolicy, ExplicitPoolsBypassSizingIndependently) { } TEST(CpuThreadPolicy, GeneralModalTransformsKeepTheirMeasuredWidth) { - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; for (const std::size_t elements : {98496U, 209304U}) { EXPECT_EQ(native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, @@ -2930,7 +2979,7 @@ TEST(CpuThreadPolicy, MetadataNamesEveryOperationClass) { } TEST(CpuThreadPolicy, ACompactTeamScopeHoldsAnOpenMpBlasAtOneThread) { -#if !defined(KOKKOS_ENABLE_OPENMP) +#ifndef KOKKOS_ENABLE_OPENMP GTEST_SKIP() << "the hold is an OpenMP build's"; #else const int before = omp_get_max_threads(); @@ -3004,12 +3053,13 @@ TEST(CpuThreadPolicy, TransformBatchesThreadByRowsAndRowLength) { if (!native::cpu_size_policy_enabled) GTEST_SKIP() << "needs automatic per-operation sizing"; native::CpuExecution execution = native::test::table(); - execution.grain[static_cast(native::CpuOperation::fft)] = {8192, - 1U << 22}; + execution.grain[static_cast(native::CpuOperation::fft)] = { + .threshold = 8192, .full = 1U << 22}; execution.fft_thread_elements = 4096; execution.fft_long_row = 16384; execution.medium_threads = 4; - const native::ThreadPolicy policy{18, 18, 18, 18}; + const native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; const auto threads = [&](const std::size_t rows, const std::size_t length) { return native::operation_threads(policy, execution, native::CpuPool::fft, native::CpuOperation::fft, rows * length, length); @@ -3069,25 +3119,25 @@ TEST(CpuThreadPolicy, CompactProblemsHaveOneColumnOnOneNode) { const auto base = general_problem(native::FieldFamily::field_real); EXPECT_TRUE(native::compact_problem(base)); auto nodes = base; - nodes.n_nodes = 2; + nodes.dims.n_nodes = 2; EXPECT_FALSE(native::compact_problem(nodes)); auto modes = base; - modes.n_modes = 2; + modes.dims.n_modes = 2; EXPECT_FALSE(native::compact_problem(modes)); auto pairs = base; - pairs.n_pol_transform = 2; + pairs.dims.n_pol_transform = 2; EXPECT_FALSE(native::compact_problem(pairs)); auto long_column = base; - long_column.n_to = native::test::table().compact_max_fine_samples; + long_column.dims.n_to = native::test::table().compact_max_fine_samples; EXPECT_FALSE(native::compact_problem(long_column)); auto modal = base; - modal.general_modal = true; + modal.general.enabled = true; EXPECT_FALSE(native::compact_problem(modal)); auto flow = base; - flow.coupled_flow = true; + flow.flow.enabled = true; EXPECT_FALSE(native::compact_problem(flow)); auto free_space = base; - free_space.n_x = 2; + free_space.spatial.n_x = 2; EXPECT_FALSE(native::compact_problem(free_space)); } @@ -3108,15 +3158,15 @@ TEST(CpuThreadPolicy, MetadataDistinguishesEqualWorkWithDifferentResponseShapes) TEST(CpuThreadPolicy, MatrixPackingDoesNotInheritTheDenseWorkWidth) { auto problem = general_problem(native::FieldFamily::field_real); - problem.n_to = 32768; - problem.n_nodes = 35; - problem.node_chunk = 33; - problem.synthesis_matrix.assign(problem.n_nodes, 1.0); - problem.projection_matrix.assign(problem.n_nodes, 1.0 / 35.0); - problem.projection_coarse_matrix = problem.projection_matrix; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_to = 32768; + problem.dims.n_nodes = 35; + problem.dims.node_chunk = 33; + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes, 1.0); + problem.transform.projection_matrix.assign(problem.dims.n_nodes, 1.0 / 35.0); + problem.transform.projection_coarse_matrix = problem.transform.projection_matrix; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); native::GeneralState state(problem); const auto sizing = state.execution_sizing(); // Full and tail matrices contain 33 and 2 entries, or twice those counts @@ -3137,21 +3187,21 @@ TEST(CpuThreadPolicy, WorkspacesKeepIndependentChoicesAndReleaseTheirPools) { auto problem = general_problem(native::FieldFamily::envelope); native::GeneralState small(problem); // One long row, past the compact bound: its transforms thread by shape. - problem.n_to = 65536; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_to = 65536; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); native::GeneralState large(problem); native::GeneralState another_small(general_problem(native::FieldFamily::envelope)); const auto small_sizing = small.execution_sizing(); EXPECT_EQ(small_sizing, another_small.execution_sizing()); - const auto key = "fft.fft." + std::to_string(problem.n_to) + ".samples." + - std::to_string(problem.n_to); - EXPECT_EQ(large.execution_sizing().at(key).at("elements"), problem.n_to); + const auto key = "fft.fft." + std::to_string(problem.dims.n_to) + ".samples." + + std::to_string(problem.dims.n_to); + EXPECT_EQ(large.execution_sizing().at(key).at("elements"), problem.dims.n_to); EXPECT_EQ(large.execution_sizing().at(key).at("threads"), static_cast(native::operation_threads( native::thread_policy(), native::test::table(), native::CpuPool::fft, - native::CpuOperation::fft, problem.n_to, problem.n_to))); + native::CpuOperation::fft, problem.dims.n_to, problem.dims.n_to))); large.close(); EXPECT_EQ(small.execution_sizing(), small_sizing); small.close(); diff --git a/native/tests/native/test_coupled_flow.cpp b/native/tests/native/test_coupled_flow.cpp index c7f2c401f..a2b1c9afd 100644 --- a/native/tests/native/test_coupled_flow.cpp +++ b/native/tests/native/test_coupled_flow.cpp @@ -2,12 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/coupled_flow.hpp" #include "solprop_native/flow_functions.hpp" -#include "execution.hpp" - #include #include @@ -44,7 +43,7 @@ double fill(const std::size_t index, const double scale) { } // Bins 1 and 3 carry the block; 0 and 2 have none, and the action is the identity. -const std::vector kSlot{0, 1, 0, 2}; +constexpr std::array kSlot{0, 1, 0, 2}; constexpr std::size_t kActive = 2; std::size_t block(const std::size_t count) { return count * kActive * kModes * kModes; } @@ -52,45 +51,45 @@ std::size_t block(const std::size_t count) { return count * kActive * kModes * k native::Problem flow_problem(const bool residual) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = 4; - problem.n_to = 8; - problem.n_omega = kOmega; - problem.n_stages = kStages; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.n_modes = kModes; - problem.family = native::FieldFamily::field_real; - problem.crop_index = {0, 1, 2, 3}; - problem.active_mask.assign(kOmega, 1U); - problem.norm_mask.assign(kOmega, 1U); - problem.omega_window.assign(kOmega, 1.0); - problem.omega = {1.0, 2.0, 3.0, 4.0}; - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_t = 4; + problem.dims.n_to = 8; + problem.dims.n_omega = kOmega; + problem.stepper.n_stages = kStages; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.n_modes = kModes; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index = {0, 1, 2, 3}; + problem.grid.active_mask.assign(kOmega, 1U); + problem.grid.norm_mask.assign(kOmega, 1U); + problem.grid.omega_window.assign(kOmega, 1.0); + problem.grid.omega = {1.0, 2.0, 3.0, 4.0}; + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); // A mode-independent carrier, as the flow requires: exp(dPhi) is a scalar per bin. for (std::size_t mode = 0; mode < kModes; ++mode) for (std::size_t index = 0; index < kOmega; ++index) - problem.phi_terms.emplace_back(-0.3, 5.0 * static_cast(index)); - problem.prefactor_terms.assign(kModes * kOmega, Complex(1.0, 0.0)); + problem.linear.phi_terms.emplace_back(-0.3, 5.0 * static_cast(index)); + problem.linear.prefactor_terms.assign(kModes * kOmega, Complex(1.0, 0.0)); // Heun's pair with a third stage at c = 1: a zero first fraction and a repeated // whole one, the two cases the action table has. - problem.tableau_matrix = {0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.5, 0.5, 0.0}; - problem.tableau_nodes = {0.0, 1.0, 1.0}; - problem.solution_weights = {0.5, 0.5, 0.0}; - problem.error_weights = {0.5, 0.5, -1.0}; - problem.synthesis_matrix.assign(kModes, 0.0); - problem.projection_matrix.assign(kModes, 0.0); - problem.coupled_flow = true; - problem.flow_active = kActive; - problem.flow_sections = kSections; - problem.flow_slot = kSlot; + problem.stepper.tableau_matrix = {0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.5, 0.5, 0.0}; + problem.stepper.tableau_nodes = {0.0, 1.0, 1.0}; + problem.stepper.solution_weights = {0.5, 0.5, 0.0}; + problem.stepper.error_weights = {0.5, 0.5, -1.0}; + problem.transform.synthesis_matrix.assign(kModes, 0.0); + problem.transform.projection_matrix.assign(kModes, 0.0); + problem.flow.enabled = true; + problem.flow.active = kActive; + problem.flow.sections = kSections; + problem.flow.slot.assign(kSlot.begin(), kSlot.end()); if (residual) { - problem.flow_terms = kTerms; + problem.flow.terms = kTerms; for (std::size_t index = 0; index < block(kSections); ++index) - problem.flow_frozen.emplace_back(fill(index + 3, 2.0), fill(index + 5, 1.5)); + problem.flow.frozen.emplace_back(fill(index + 3, 2.0), fill(index + 5, 1.5)); for (std::size_t index = 0; index < block(kTerms); ++index) - problem.flow_residual_terms.emplace_back(fill(index + 7, 2.5), + problem.flow.residual_terms.emplace_back(fill(index + 7, 2.5), fill(index + 11, 1.0)); } return problem; @@ -114,21 +113,21 @@ Actions actions() { native::TrialScalars scalars(const bool residual, const std::size_t section) { native::TrialScalars result{ - std::vector((kStages + 1), kStep), - std::vector(kStages, kStep), - std::vector(kStages, 1.0), - std::vector(kStages, 1.0), - std::vector(kStages, 1.0), - {}, - {}, - {}, - {}, - {}, - {}, - section, - {}, - {}, - {}, + .phi_delta = std::vector((kStages + 1), kStep), + .phi_at = std::vector(kStages, kStep), + .prefactor = std::vector(kStages, 1.0), + .density = std::vector(kStages, 1.0), + .taper = std::vector(kStages, 1.0), + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = section, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}, }; if (residual) for (std::size_t index = 0; index < kStages * kTerms; ++index) @@ -138,6 +137,7 @@ native::TrialScalars scalars(const bool residual, const std::size_t section) { std::vector initial() { std::vector field; + field.reserve(kModes * kOmega); for (std::size_t index = 0; index < kModes * kOmega; ++index) field.emplace_back(fill(index + 41, 1.0), fill(index + 43, 1.0)); return field; @@ -164,13 +164,13 @@ std::vector reference_step(const native::Problem& problem, const Action std::vector solution(current.size()); const auto phase = [&](const std::size_t stage, const std::size_t mode, const std::size_t index) { - return trial.phi_delta[stage] * problem.phi_terms[mode * kOmega + index]; + return trial.phi_delta[stage] * problem.linear.phi_terms[mode * kOmega + index]; }; for (std::size_t stage = 0; stage < kStages; ++stage) { std::vector sum(current.size()); for (std::size_t previous = 0; previous < stage; ++previous) for (std::size_t element = 0; element < sum.size(); ++element) - sum[element] += problem.tableau_matrix[stage * kStages + previous] * + sum[element] += problem.stepper.tableau_matrix[stage * kStages + previous] * stages[previous][element]; std::vector lab(current.size()); std::vector drive(current.size()); @@ -191,7 +191,7 @@ std::vector reference_step(const native::Problem& problem, const Action lab[row * kOmega + index] = cexp(phase(stage, row, index)) * (state + kStep * carried); } - if (slot == 0 || problem.flow_terms == 0) + if (slot == 0 || problem.flow.terms == 0) continue; for (std::size_t row = 0; row < kModes; ++row) { Complex residual{}; @@ -200,13 +200,13 @@ std::vector reference_step(const native::Problem& problem, const Action for (std::size_t term = 0; term < kTerms; ++term) combined += trial.flow_weights[stage * kTerms + term] * - problem.flow_residual_terms + problem.flow.residual_terms [((term * kActive + slot - 1) * kModes + row) * kModes + column]; combined -= - problem.flow_frozen[((trial.flow_section * kActive + slot - 1) * kModes + - row) * - kModes + - column]; + problem.flow + .frozen[((trial.flow_section * kActive + slot - 1) * kModes + row) * + kModes + + column]; residual += combined * lab[column * kOmega + index]; } drive[row * kOmega + index] = cexp(-phase(stage, row, index)) * residual; @@ -227,7 +227,8 @@ std::vector reference_step(const native::Problem& problem, const Action } } for (std::size_t element = 0; element < solution.size(); ++element) - solution[element] += problem.solution_weights[stage] * stage_value[element]; + solution[element] += + problem.stepper.solution_weights[stage] * stage_value[element]; stages.push_back(stage_value); } std::vector advanced(current.size()); @@ -291,9 +292,9 @@ TEST(CoupledFlow, ConstantBlockCarriesTheStepByItsWholeAction) { for (std::size_t row = 0; row < kModes; ++row) { const auto element = row * kOmega; EXPECT_NE(actual[element + 1], - start[element + 1] * cexp(kStep * problem.phi_terms[1])); + start[element + 1] * cexp(kStep * problem.linear.phi_terms[1])); EXPECT_LE(Kokkos::abs(actual[element] - - start[element] * cexp(kStep * problem.phi_terms[0])), + start[element] * cexp(kStep * problem.linear.phi_terms[0])), kBound * largest(expected)); } } @@ -326,7 +327,7 @@ TEST(CoupledFlow, RepeatedTrialsAgreeToRounding) { TEST(CoupledFlow, NoStageIsReusedAcrossSteps) { auto problem = flow_problem(false); - problem.first_same_as_last = true; + problem.stepper.first_same_as_last = true; const auto flow = actions(); native::GeneralState state(problem); state.set_initial(initial(), {}); @@ -339,25 +340,25 @@ TEST(CoupledFlow, NoStageIsReusedAcrossSteps) { TEST(CoupledFlow, MalformedDataAreRefusedBeforeAllocation) { auto slot = flow_problem(false); - slot.flow_slot = {0, 2, 0, 1}; + slot.flow.slot = {0, 2, 0, 1}; EXPECT_THROW(native::GeneralState{slot}, std::invalid_argument); auto count = flow_problem(false); - count.flow_active = 3; + count.flow.active = 3; EXPECT_THROW(native::GeneralState{count}, std::invalid_argument); auto pol = flow_problem(false); - pol.n_pol_state = 2; + pol.dims.n_pol_state = 2; EXPECT_THROW(native::GeneralState{pol}, std::invalid_argument); auto noise = flow_problem(false); - noise.modified_noise = true; + noise.stepper.modified_noise = true; EXPECT_THROW(native::GeneralState{noise}, std::invalid_argument); auto terms = flow_problem(true); - terms.flow_residual_terms.pop_back(); + terms.flow.residual_terms.pop_back(); EXPECT_THROW(native::GeneralState{terms}, std::invalid_argument); auto stray = flow_problem(false); - stray.coupled_flow = false; + stray.flow.enabled = false; EXPECT_THROW(native::GeneralState{stray}, std::invalid_argument); auto nonfinite = flow_problem(true); - nonfinite.flow_frozen[0] = Complex(std::numeric_limits::quiet_NaN(), 0.0); + nonfinite.flow.frozen[0] = Complex(std::numeric_limits::quiet_NaN(), 0.0); EXPECT_THROW(native::GeneralState{nonfinite}, std::invalid_argument); } @@ -369,9 +370,9 @@ TEST(CoupledFlow, ATwoComponentSynthesisLeavesTheFlowAsItIs) { for (const bool residual : {false, true}) { const auto scalar = flow_problem(residual); auto vector = scalar; - vector.n_pol_transform = 2; - vector.synthesis_matrix.assign(2 * kModes, 0.0); - vector.projection_matrix.assign(2 * kModes, 0.0); + vector.dims.n_pol_transform = 2; + vector.transform.synthesis_matrix.assign(2 * kModes, 0.0); + vector.transform.projection_matrix.assign(2 * kModes, 0.0); const auto flow = actions(); const auto trial = scalars(residual, 1); const auto expected = native_step(scalar, flow, trial); @@ -423,10 +424,10 @@ TEST(CoupledFlow, LedgerCountsTheFlowViewsAndIsZeroElsewhere) { "flow_carried_solution", "flow_carried_error"}) EXPECT_EQ(ledger.at(name), kModes * kOmega * complex_bytes) << name; auto plain = flow_problem(false); - plain.coupled_flow = false; - plain.flow_active = 0; - plain.flow_sections = 0; - plain.flow_slot.clear(); + plain.flow.enabled = false; + plain.flow.active = 0; + plain.flow.sections = 0; + plain.flow.slot.clear(); native::GeneralState other(plain); for (const auto& [name, bytes] : native::CoupledFlowAction::empty_ledger()) EXPECT_EQ(other.memory_ledger().at(name), 0U) << name; @@ -442,9 +443,9 @@ TEST(CoupledFlow, ZeroWeightTermsAreSkippedWithoutMovingABit) { const auto flow = actions(); const auto trial = scalars(true, 1); auto padded = problem; - padded.flow_terms = 2 * kTerms; + padded.flow.terms = 2 * kTerms; for (std::size_t index = 0; index < block(kTerms); ++index) - padded.flow_residual_terms.emplace_back(fill(index + 53, 9.0), + padded.flow.residual_terms.emplace_back(fill(index + 53, 9.0), fill(index + 59, -7.0)); auto padded_trial = trial; padded_trial.flow_weights.clear(); @@ -559,26 +560,26 @@ native::Problem forming_problem(const bool fallback) { const auto upper = triangular(section, bin); const auto [left, right] = eigenbasis(vectors, upper); for (std::size_t mode = 0; mode < kModes; ++mode) - problem.flow_eigenvalues.push_back(upper[mode * kModes + mode]); + problem.flow.eigenvalues.push_back(upper[mode * kModes + mode]); if (fallback) { - problem.flow_route.push_back(++route); - problem.flow_left.insert(problem.flow_left.end(), kModes * kModes, Complex{}); - problem.flow_right.insert(problem.flow_right.end(), kModes * kModes, Complex{}); - problem.flow_fallback_vectors.insert(problem.flow_fallback_vectors.end(), + problem.flow.route.push_back(++route); + problem.flow.left.insert(problem.flow.left.end(), kModes * kModes, Complex{}); + problem.flow.right.insert(problem.flow.right.end(), kModes * kModes, Complex{}); + problem.flow.fallback_vectors.insert(problem.flow.fallback_vectors.end(), vectors.begin(), vectors.end()); - problem.flow_fallback_triangular.insert(problem.flow_fallback_triangular.end(), + problem.flow.fallback_triangular.insert(problem.flow.fallback_triangular.end(), upper.begin(), upper.end()); } else { - problem.flow_route.push_back(0); - problem.flow_left.insert(problem.flow_left.end(), left.begin(), left.end()); - problem.flow_right.insert(problem.flow_right.end(), right.begin(), right.end()); + problem.flow.route.push_back(0); + problem.flow.left.insert(problem.flow.left.end(), left.begin(), left.end()); + problem.flow.right.insert(problem.flow.right.end(), right.begin(), right.end()); } } return problem; } // Distinct lengths, a zero one and a repeated whole one. -const std::vector kFractions{0.0, 0.5, 1.0, 1.0}; +std::vector fractions() { return {0.0, 0.5, 1.0, 1.0}; } constexpr double kLength = 0.02; // |l lambda| up to 1.9: the squarings are exercised struct Formed { @@ -589,7 +590,7 @@ struct Formed { Formed form(const native::Problem& problem, const std::size_t section) { native::GeneralState state(problem); Formed result; - result.measured = state.form_flow_actions(section, kLength, kFractions); + result.measured = state.form_flow_actions(section, kLength, fractions()); result.forward = state.flow_actions_to_host(false); result.inverse = state.flow_actions_to_host(true); return result; @@ -620,7 +621,7 @@ TEST(CoupledFlowFormation, BothRoutesFormTheSameActions) { for (const std::size_t section : {std::size_t{0}, std::size_t{1}}) { const auto eigen = form(forming_problem(false), section); const auto squared = form(forming_problem(true), section); - for (std::size_t slot = 1; slot < kFractions.size(); ++slot) + for (std::size_t slot = 1; slot < fractions().size(); ++slot) for (std::size_t bin = 0; bin < kActive; ++bin) { const auto forward = block_at(eigen.forward, slot, bin); const auto inverse = block_at(eigen.inverse, slot, bin); @@ -673,7 +674,7 @@ TEST(CoupledFlowFormation, FormedActionsCarryATrialAsLoadedOnesDo) { const auto trial = scalars(false, 1); native::GeneralState state(problem); state.set_initial(initial(), {}); - static_cast(state.form_flow_actions(1, kLength, kFractions)); + static_cast(state.form_flow_actions(1, kLength, fractions())); static_cast(state.trial_step(kStep, trial)); state.finish(kStep); // The same trial on the same actions, formed here and loaded there: rounding only. @@ -685,16 +686,16 @@ TEST(CoupledFlowFormation, FormedActionsCarryATrialAsLoadedOnesDo) { TEST(CoupledFlowFormation, FormationAndItsDataAreChecked) { native::GeneralState plain(flow_problem(false)); - EXPECT_THROW(static_cast(plain.form_flow_actions(0, kLength, kFractions)), + EXPECT_THROW(static_cast(plain.form_flow_actions(0, kLength, fractions())), std::logic_error); EXPECT_THROW(static_cast(plain.flow_actions_to_host(false)), std::logic_error); native::GeneralState state(forming_problem(false)); EXPECT_THROW( - static_cast(state.form_flow_actions(kSections, kLength, kFractions)), + static_cast(state.form_flow_actions(kSections, kLength, fractions())), std::invalid_argument); EXPECT_THROW(static_cast(state.form_flow_actions(0, kLength, {0.0, 1.0})), std::invalid_argument); - EXPECT_THROW(static_cast(state.form_flow_actions(0, -kLength, kFractions)), + EXPECT_THROW(static_cast(state.form_flow_actions(0, -kLength, fractions())), std::invalid_argument); EXPECT_THROW( static_cast(state.form_flow_actions(0, kLength, {0.0, 0.5, 2.0, 1.0})), @@ -703,23 +704,23 @@ TEST(CoupledFlowFormation, FormationAndItsDataAreChecked) { static_cast(state.form_flow_actions(0, kLength, {0.0, 0.5, 1.0, 0.0})), std::invalid_argument); auto numbering = forming_problem(true); - numbering.flow_route[0] = 2; + numbering.flow.route[0] = 2; EXPECT_THROW(native::GeneralState{numbering}, std::invalid_argument); auto fallback = forming_problem(true); - fallback.flow_fallback_vectors.pop_back(); + fallback.flow.fallback_vectors.pop_back(); EXPECT_THROW(native::GeneralState{fallback}, std::invalid_argument); auto partial = forming_problem(false); - partial.flow_right.pop_back(); + partial.flow.right.pop_back(); EXPECT_THROW(native::GeneralState{partial}, std::invalid_argument); auto nonfinite = forming_problem(false); - nonfinite.flow_eigenvalues[0] = Complex(std::numeric_limits::infinity(), 0.0); + nonfinite.flow.eigenvalues[0] = Complex(std::numeric_limits::infinity(), 0.0); EXPECT_THROW(native::GeneralState{nonfinite}, std::invalid_argument); auto stray = flow_problem(false); - stray.coupled_flow = false; - stray.flow_active = 0; - stray.flow_sections = 0; - stray.flow_slot.clear(); - stray.flow_route = {0}; + stray.flow.enabled = false; + stray.flow.active = 0; + stray.flow.sections = 0; + stray.flow.slot.clear(); + stray.flow.route = {0}; EXPECT_THROW(native::GeneralState{stray}, std::invalid_argument); } diff --git a/native/tests/native/test_crossing.cpp b/native/tests/native/test_crossing.cpp index 46e92eae5..ee9b3f998 100644 --- a/native/tests/native/test_crossing.cpp +++ b/native/tests/native/test_crossing.cpp @@ -31,7 +31,8 @@ class RingSizes : public ::testing::TestWithParam {}; TEST_P(RingSizes, CrossesEveryByteBothWays) { const auto source = counting(GetParam()); - std::vector down(source.size(), 0), up(source.size(), 0); + std::vector down(source.size(), 0); + std::vector up(source.size(), 0); Ring ring; ring.to_host(down.data(), source.data(), source.size()); ring.from_host(up.data(), down.data(), down.size()); diff --git a/native/tests/native/test_dispersive.cpp b/native/tests/native/test_dispersive.cpp index d024df28a..6f75db02c 100644 --- a/native/tests/native/test_dispersive.cpp +++ b/native/tests/native/test_dispersive.cpp @@ -2,13 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dispersive.hpp" -#include "execution.hpp" - #include - #include #include @@ -53,7 +51,8 @@ TEST(DispersiveBodies, TheContractionIsTheIsotropicTensorSum) { const Complex z[2] = {{0.9, 0.1}, {-0.2, 0.8}}; Complex expected[2]; tensor_sum(x, y, z, expected); - Complex out0, out1; + Complex out0; + Complex out1; native::dispersive_isotropic(x[0], x[1], y[0], y[1], z[0], z[1], out0, out1); EXPECT_LT(distance(out0, expected[0]), 16 * kEpsilon); EXPECT_LT(distance(out1, expected[1]), 16 * kEpsilon); @@ -71,7 +70,9 @@ TEST(DispersiveBodies, TheScalarClosedFormsAreTheGeneralGroups) { const Complex f{-1.2, 0.35}; const Complex carrier{std::cos(0.7), std::sin(0.7)}; const Complex zero{}; - Complex general, unused, harmonic; + Complex general; + Complex unused; + Complex harmonic; native::dispersive_conserving(u, zero, u, zero, u, zero, general, unused); EXPECT_LT(distance(native::dispersive_conserving_self(u), general), 16 * kEpsilon); native::dispersive_conserving(u, zero, u, zero, f, zero, general, unused); @@ -98,8 +99,10 @@ TEST(DispersiveBodies, TheScalarClosedFormsAreTheGeneralGroups) { TEST(DispersiveBodies, TheConservingGroupIsTheQuartetsWithTwoPositiveLegs) { // `|u|^2 u` contracted isotropically: `(u . u*) u` twice and `(u . u) u*` once, the // number-conserving group of `(u, u, u)` (SEP 6 Phase 0). - const Complex u0{0.4, 0.9}, u1{-0.7, 0.2}; - Complex out0, out1; + const Complex u0{0.4, 0.9}; + const Complex u1{-0.7, 0.2}; + Complex out0; + Complex out1; native::dispersive_conserving(u0, u1, u0, u1, u0, u1, out0, out1); const Complex square = u0 * u0 + u1 * u1; const double modulus = @@ -140,10 +143,13 @@ TEST(DispersiveBodies, TheThreeComponentContractionIsTheTensorSum) { native::dispersive_isotropic_n(x, y, z, out); for (int c = 0; c < 3; ++c) EXPECT_LT(distance(out[c], expected[c]), 32 * kEpsilon) << c; - const Complex x2[2] = {x[0], x[1]}, y2[2] = {y[0], y[1]}, z2[2] = {z[0], z[1]}; + const Complex x2[2] = {x[0], x[1]}; + const Complex y2[2] = {y[0], y[1]}; + const Complex z2[2] = {z[0], z[1]}; Complex two[2]; native::dispersive_isotropic_n(x2, y2, z2, two); - Complex out0, out1; + Complex out0; + Complex out1; native::dispersive_isotropic(x[0], x[1], y[0], y[1], z[0], z[1], out0, out1); // Each body is within 16 eps of the tensor sum (the test above), so 32 eps of the // other. @@ -171,13 +177,18 @@ TEST(DispersiveBodies, TheDistinctLegGroupsAreTheirTensorSums) { const Complex x[3] = {{0.3, -1.1}, {0.7, 0.2}, {-0.5, 0.4}}; const Complex y[3] = {{-0.4, 0.5}, {1.3, -0.6}, {0.2, 0.9}}; const Complex z[3] = {{0.9, 0.1}, {-0.2, 0.8}, {0.6, -0.3}}; - Complex xc[3], yc[3], zc[3]; + Complex xc[3]; + Complex yc[3]; + Complex zc[3]; for (int c = 0; c < 3; ++c) { xc[c] = Kokkos::conj(x[c]); yc[c] = Kokkos::conj(y[c]); zc[c] = Kokkos::conj(z[c]); } - Complex a[3], b[3], d[3], out[3]; + Complex a[3]; + Complex b[3]; + Complex d[3]; + Complex out[3]; tensor_sum3(x, y, zc, a); tensor_sum3(x, yc, z, b); tensor_sum3(xc, y, z, d); @@ -185,10 +196,18 @@ TEST(DispersiveBodies, TheDistinctLegGroupsAreTheirTensorSums) { for (int c = 0; c < 3; ++c) EXPECT_LT(distance(out[c], a[c] + b[c] + d[c]), 3 * 32 * kEpsilon) << c; - const Complex x2[2] = {x[0], x[1]}, y2[2] = {y[0], y[1]}, z2[2] = {z[0], z[1]}; - const Complex xc2[2] = {xc[0], xc[1]}, yc2[2] = {yc[0], yc[1]}, - zc2[2] = {zc[0], zc[1]}; - Complex p[2], q[2], r[2], u[2], out0, out1; + const Complex x2[2] = {x[0], x[1]}; + const Complex y2[2] = {y[0], y[1]}; + const Complex z2[2] = {z[0], z[1]}; + const Complex xc2[2] = {xc[0], xc[1]}; + const Complex yc2[2] = {yc[0], yc[1]}; + const Complex zc2[2] = {zc[0], zc[1]}; + Complex p[2]; + Complex q[2]; + Complex r[2]; + Complex u[2]; + Complex out0; + Complex out1; tensor_sum(x2, y2, zc2, p); tensor_sum(x2, yc2, z2, q); tensor_sum(xc2, y2, z2, r); @@ -280,43 +299,43 @@ TEST(DispersiveMixture, TheRowsAreTheReferenceArithmetic) { native::Problem dispersive_problem() { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = 4; - problem.n_to = 8; - problem.n_omega = 3; - problem.n_stages = 1; - problem.family = native::FieldFamily::field_real; - problem.crop_index = {0, 1, 2}; - problem.active_mask = {1, 1, 1}; - problem.norm_mask = {1, 1, 1}; - problem.omega_window = {1.0, 1.0, 1.0}; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex{1.0, 0.0}); - problem.phi_terms.assign(problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_omega, Complex{1.0, 0.0}); - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.0}; - problem.synthesis_matrix = {1.0}; - problem.projection_matrix = {1.0}; - problem.synthesis = 1.0; - problem.projection = 1.0; - const std::size_t bins = problem.n_to / 2 + 1; - problem.dispersive = true; - problem.dispersive_symmetric = true; - problem.dispersive_form = 0; - problem.dispersive_inputs.assign(bins, 1.0); - problem.dispersive_term_multipliers.assign(bins, 1.0); - problem.dispersive_term_rows = {0}; - problem.dispersive_term_channels = {0}; - problem.dispersive_coefficients = {2.0}; - problem.dispersive_pair_constants = {0.0, 0.0}; - problem.dispersive_group_offsets = {0, 1}; - problem.dispersive_group_members = {0}; - problem.dispersive_input_scale.assign(bins, 1.0); - problem.dispersive_output_scale.assign(bins, 1.0); + problem.dims.n_t = 4; + problem.dims.n_to = 8; + problem.dims.n_omega = 3; + problem.stepper.n_stages = 1; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index = {0, 1, 2}; + problem.grid.active_mask = {1, 1, 1}; + problem.grid.norm_mask = {1, 1, 1}; + problem.grid.omega_window = {1.0, 1.0, 1.0}; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex{1.0, 0.0}); + problem.linear.phi_terms.assign(problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_omega, Complex{1.0, 0.0}); + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.0}; + problem.transform.synthesis_matrix = {1.0}; + problem.transform.projection_matrix = {1.0}; + problem.transform.synthesis = 1.0; + problem.transform.projection = 1.0; + const std::size_t bins = problem.dims.n_to / 2 + 1; + problem.dispersive.enabled = true; + problem.dispersive.symmetric = true; + problem.dispersive.form = 0; + problem.dispersive.inputs.assign(bins, 1.0); + problem.dispersive.term_multipliers.assign(bins, 1.0); + problem.dispersive.term_rows = {0}; + problem.dispersive.term_channels = {0}; + problem.dispersive.coefficients = {2.0}; + problem.dispersive.pair_constants = {0.0, 0.0}; + problem.dispersive.group_offsets = {0, 1}; + problem.dispersive.group_members = {0}; + problem.dispersive.input_scale.assign(bins, 1.0); + problem.dispersive.output_scale.assign(bins, 1.0); return problem; } @@ -341,7 +360,7 @@ TEST(DispersiveResponse, AUnitFilterIsTheConstantCubicOfOneNode) { static_cast(state.trial_step(0.0, scalars)); const auto projected = state.projected_to_host(false); // The field in time, cubed, transformed back. - const std::size_t n_to = problem.n_to; + const std::size_t n_to = problem.dims.n_to; std::vector time(n_to, 0.0); for (std::size_t sample = 0; sample < n_to; ++sample) { double value = field[0].real() / static_cast(n_to); @@ -395,45 +414,45 @@ TEST(DispersiveResponse, ATrialNeedsEachStageSDensities) { TEST(DispersiveResponse, MalformedLayoutsAreRefusedBeforeAllocation) { { auto problem = dispersive_problem(); - problem.dispersive_form = 1; // the conserving sector on the real family + problem.dispersive.form = 1; // the conserving sector on the real family EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_input_scale.pop_back(); + problem.dispersive.input_scale.pop_back(); EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_term_rows = {1}; // a product the layout does not have + problem.dispersive.term_rows = {1}; // a product the layout does not have EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_group_members = {1}; + problem.dispersive.group_members = {1}; EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive = false; // data without the response + problem.dispersive.enabled = false; // data without the response EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } } TEST(DispersiveResponse, AProblemWithoutItNamesTheRowsAtZero) { auto problem = dispersive_problem(); - problem.dispersive = false; - problem.dispersive_inputs.clear(); - problem.dispersive_term_multipliers.clear(); - problem.dispersive_term_rows.clear(); - problem.dispersive_term_channels.clear(); - problem.dispersive_coefficients.clear(); - problem.dispersive_pair_constants.clear(); - problem.dispersive_group_offsets.clear(); - problem.dispersive_group_members.clear(); - problem.dispersive_input_scale.clear(); - problem.dispersive_output_scale.clear(); - problem.nonlinear = true; + problem.dispersive.enabled = false; + problem.dispersive.inputs.clear(); + problem.dispersive.term_multipliers.clear(); + problem.dispersive.term_rows.clear(); + problem.dispersive.term_channels.clear(); + problem.dispersive.coefficients.clear(); + problem.dispersive.pair_constants.clear(); + problem.dispersive.group_offsets.clear(); + problem.dispersive.group_members.clear(); + problem.dispersive.input_scale.clear(); + problem.dispersive.output_scale.clear(); + problem.kerr.nonlinear = true; native::GeneralState state(problem); const auto ledger = state.memory_ledger(); for (const auto& [name, bytes] : native::DispersiveResponse::empty_ledger()) @@ -446,39 +465,39 @@ native::Problem general_problem(const bool symmetric) { // krypton in region 5, as solprop's leaf groups them. native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_to = 8; - problem.n_stages = 1; - problem.family = native::FieldFamily::field_real; - problem.general_modal = true; - problem.general_nodes = 5; - problem.general_node_chunk = 2; - problem.general_components = 3; - const std::size_t bins = problem.n_to / 2 + 1; - problem.dispersive = true; - problem.dispersive_symmetric = symmetric; - problem.dispersive_form = 1; - problem.dispersive_coefficients = {1.0, 2.0, 3.0}; - problem.dispersive_pair_constants.assign(6, 0.5); - problem.dispersive_input_scale.assign(bins, 1.0); - problem.dispersive_output_scale.assign(bins, 0.125); - problem.dispersive_node_region = {5, 5, 3, 0, 3}; + problem.dims.n_to = 8; + problem.stepper.n_stages = 1; + problem.dims.family = native::FieldFamily::field_real; + problem.general.enabled = true; + problem.general.nodes = 5; + problem.general.node_chunk = 2; + problem.general.components = 3; + const std::size_t bins = problem.dims.n_to / 2 + 1; + problem.dispersive.enabled = true; + problem.dispersive.symmetric = symmetric; + problem.dispersive.form = 1; + problem.dispersive.coefficients = {1.0, 2.0, 3.0}; + problem.dispersive.pair_constants.assign(6, 0.5); + problem.dispersive.input_scale.assign(bins, 1.0); + problem.dispersive.output_scale.assign(bins, 0.125); + problem.dispersive.node_region = {5, 5, 3, 0, 3}; if (symmetric) { - problem.dispersive_group_offsets = {0, 1, 3}; - problem.dispersive_group_members = {0, 1, 2}; - problem.dispersive_group_keys = {3, 5}; - problem.dispersive_inputs.assign(2 * bins, 1.0); - problem.dispersive_term_rows = {0, 1}; - problem.dispersive_term_channels = {0, 4}; - problem.dispersive_term_multipliers.assign(2 * bins, 0.125); - problem.dispersive_quartics.assign(3 * bins, 1.0); + problem.dispersive.group_offsets = {0, 1, 3}; + problem.dispersive.group_members = {0, 1, 2}; + problem.dispersive.group_keys = {3, 5}; + problem.dispersive.inputs.assign(2 * bins, 1.0); + problem.dispersive.term_rows = {0, 1}; + problem.dispersive.term_channels = {0, 4}; + problem.dispersive.term_multipliers.assign(2 * bins, 0.125); + problem.dispersive.quartics.assign(3 * bins, 1.0); } else { - problem.dispersive_group_offsets = {0, 1, 2, 3}; - problem.dispersive_group_members = {0, 1, 2}; - problem.dispersive_group_keys = {3, 5, 5}; - problem.dispersive_inputs.assign(6 * bins, 1.0); - problem.dispersive_term_rows = {0, 2, 3, 1, 4, 5, 1, 6, 7}; - problem.dispersive_term_channels = {0, 0, 3, 1, 1, 3, 2, 2, 3}; - problem.dispersive_term_multipliers.assign(9 * bins, 0.125); + problem.dispersive.group_offsets = {0, 1, 2, 3}; + problem.dispersive.group_members = {0, 1, 2}; + problem.dispersive.group_keys = {3, 5, 5}; + problem.dispersive.inputs.assign(6 * bins, 1.0); + problem.dispersive.term_rows = {0, 2, 3, 1, 4, 5, 1, 6, 7}; + problem.dispersive.term_channels = {0, 0, 3, 1, 1, 3, 2, 2, 3}; + problem.dispersive.term_multipliers.assign(9 * bins, 0.125); } return problem; } @@ -505,28 +524,28 @@ TEST(DispersiveWeights, ThePolynomialFormsOneCubePerRegion) { TEST(DispersiveWeights, RegionsBelongToTheGeneralRouteAlone) { { auto problem = general_problem(false); - problem.dispersive_node_region.pop_back(); + problem.dispersive.node_region.pop_back(); EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_group_keys.pop_back(); + problem.dispersive.group_keys.pop_back(); EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_form = 2; // an envelope's harmonic groups + problem.dispersive.form = 2; // an envelope's harmonic groups EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_term_rows.back() = 8; // one product past the layout + problem.dispersive.term_rows.back() = 8; // one product past the layout EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { // Off the general route every channel acts at every node. auto problem = dispersive_problem(); - problem.dispersive_group_keys = {0}; + problem.dispersive.group_keys = {0}; EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } EXPECT_NO_THROW(native::require_dispersive(general_problem(true))); diff --git a/native/tests/native/test_envelope_plasma.cpp b/native/tests/native/test_envelope_plasma.cpp index e6aee32e6..ceebd3acd 100644 --- a/native/tests/native/test_envelope_plasma.cpp +++ b/native/tests/native/test_envelope_plasma.cpp @@ -26,8 +26,8 @@ struct PolynomialRate { }; std::array polynomial_rules(const bool inconsistent = false) { - RuleView nodes("envelope_test.nodes", 4, 512), - weights("envelope_test.weights", 4, 512); + RuleView nodes("envelope_test.nodes", 4, 512); + RuleView weights("envelope_test.weights", 4, 512); auto host_nodes = Kokkos::create_mirror_view(nodes); auto host_weights = Kokkos::create_mirror_view(weights); for (std::size_t level = 0; level < 4; ++level) { @@ -81,8 +81,10 @@ void polynomial_moments_and_directional_work() { // Two ordered sums of at most 128 terms, plus bounded elementary arithmetic. constexpr double tolerance = 1024 * std::numeric_limits::epsilon(); for (std::size_t index = 0; index < ratios.size(); ++index) { - const double ratio = ratios[index], square = ratio * ratio; - const double mc = 2.0 * (3.0 + square), ms = 2.0 * (1.0 + 3.0 * square); + const double ratio = ratios[index]; + const double square = ratio * ratio; + const double mc = 2.0 * (3.0 + square); + const double ms = 2.0 * (1.0 + 3.0 * square); EXPECT_NEAR(actual(index, 0), 2.0, tolerance); EXPECT_NEAR(actual(index, 1), ratio, tolerance); EXPECT_NEAR(actual(index, 2), mc, tolerance * mc); @@ -97,7 +99,8 @@ void polynomial_moments_and_directional_work() { } void phase_and_real_rotation() { - constexpr double phase = 0.37, angle = 0.61; + constexpr double phase = 0.37; + constexpr double angle = 0.61; const Complex common(std::cos(phase), std::sin(phase)); const Complex first = common * Complex(2.0 * std::cos(angle), -0.7 * std::sin(angle)); const Complex second = common * Complex(2.0 * std::sin(angle), 0.7 * std::cos(angle)); @@ -180,7 +183,8 @@ void scaled_magnitude_preserves_tiny_and_huge_components() { "envelope_test.scaled", Kokkos::RangePolicy(native::ExecutionSpace{}, 0, count), KOKKOS_LAMBDA(const std::size_t index) { - const double first = pairs(index, 0), second = pairs(index, 1); + const double first = pairs(index, 0); + const double second = pairs(index, 1); const auto one = native::envelope_axes(Complex(first, second), Complex{}); const auto two = native::envelope_axes(Complex(first, 0.0), Complex(second, 0.0)); diff --git a/native/tests/native/test_fft.cpp b/native/tests/native/test_fft.cpp index 67d59ff5d..a78924958 100644 --- a/native/tests/native/test_fft.cpp +++ b/native/tests/native/test_fft.cpp @@ -104,7 +104,10 @@ TEST(FftProvider, RealAndComplexBatchesPreserveSignAndNormalization) { } TEST(FftProvider, StackedTailPlansLeaveEveryBlocksPaddingUntouched) { - constexpr std::size_t length = 32, rows = 7, active = 3, blocks = 3; + constexpr std::size_t length = 32; + constexpr std::size_t rows = 7; + constexpr std::size_t active = 3; + constexpr std::size_t blocks = 3; const RealRows input("fft_contract.stacked_input", blocks * rows, length); const ComplexRows output("fft_contract.stacked_output", blocks * rows, length / 2 + 1); @@ -421,7 +424,7 @@ TEST(RamanFftProvider, PreservesRawNormalizationInputAndPartialRows) { row < tail ? fft_bound(length, length) : 0.0); } } -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE const bool neon = std::string_view(fftw_version).find("-neon") != std::string_view::npos; const bool accelerated = length != 15 && (neon ? threads > 1 && length <= 16384 diff --git a/native/tests/native/test_general_modal.cpp b/native/tests/native/test_general_modal.cpp index 6515fc636..adfc3b70e 100644 --- a/native/tests/native/test_general_modal.cpp +++ b/native/tests/native/test_general_modal.cpp @@ -2,13 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/general_modal.hpp" -#include "execution.hpp" - #include - #include #include @@ -63,67 +61,69 @@ native::Problem modal_problem(const bool real_form, const std::size_t chunk, const std::size_t n_to = 8) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = n_to / 2; - problem.n_to = n_to; - problem.n_omega = n_omega; - problem.n_stages = 1; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.n_modes = kModes; - problem.family = native::FieldFamily::field_real; - problem.crop_index.resize(n_omega); - std::iota(problem.crop_index.begin(), problem.crop_index.end(), offset); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); + problem.dims.n_t = n_to / 2; + problem.dims.n_to = n_to; + problem.dims.n_omega = n_omega; + problem.stepper.n_stages = 1; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.n_modes = kModes; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index.resize(n_omega); + std::iota(problem.grid.crop_index.begin(), problem.grid.crop_index.end(), offset); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); for (std::size_t w = 0; w < n_omega; ++w) - problem.omega.push_back(static_cast(w + 1)); - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(kModes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(kModes * problem.n_omega, Complex(1.0, 0.0)); - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.25}; - problem.synthesis_matrix = {0.0, 0.0}; - problem.projection_matrix = {0.0, 0.0}; - problem.general_modal = true; - problem.general_real_form = real_form; - problem.general_components = kComponents; - problem.general_nodes = kNodes; - problem.general_node_chunk = chunk; - problem.general_pad_scale = 2.0; - problem.general_epsilon0 = 1.0; - const auto maps = kNodes * kComponents * kModes * problem.n_omega; + problem.grid.omega.push_back(static_cast(w + 1)); + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(kModes * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(kModes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.25}; + problem.transform.synthesis_matrix = {0.0, 0.0}; + problem.transform.projection_matrix = {0.0, 0.0}; + problem.general.enabled = true; + problem.general.real_form = real_form; + problem.general.components = kComponents; + problem.general.nodes = kNodes; + problem.general.node_chunk = chunk; + problem.general.pad_scale = 2.0; + problem.general.epsilon0 = 1.0; + const auto maps = kNodes * kComponents * kModes * problem.dims.n_omega; for (std::size_t index = 0; index < maps; ++index) { - problem.general_synthesis.emplace_back(fill(index, 1.0), fill(index + 97, 0.5)); - problem.general_projection.emplace_back(fill(index + 13, 0.8), + problem.general.synthesis.emplace_back(fill(index, 1.0), fill(index + 97, 0.5)); + problem.general.projection.emplace_back(fill(index + 13, 0.8), fill(index + 41, 0.6)); } for (std::size_t index = 0; index < kNodes * fourth(kComponents); ++index) - problem.general_tensor.push_back(fill(index + 7, 1.0)); + problem.general.tensor.push_back(fill(index + 7, 1.0)); for (std::size_t w = 0; w < n_omega; ++w) - problem.general_frequency_factor.emplace_back(0.0, -static_cast(w + 1)); + problem.general.frequency_factor.emplace_back(0.0, -static_cast(w + 1)); if (delayed) { - problem.general_responses = kResponses; - problem.general_terms = kTerms; + problem.general.responses = kResponses; + problem.general.terms = kTerms; for (std::size_t index = 0; index < kResponses * kTerms * fourth(kComponents); ++index) - problem.general_delayed_tensor.push_back(fill(index + 3, 0.5)); - for (std::size_t index = 0; index < kResponses * kTerms * (problem.n_to + 1); + problem.general.delayed_tensor.push_back(fill(index + 3, 0.5)); + for (std::size_t index = 0; index < kResponses * kTerms * (problem.dims.n_to + 1); ++index) - problem.general_delayed_spectrum.emplace_back(fill(index + 5, 1.0), + problem.general.delayed_spectrum.emplace_back(fill(index + 5, 1.0), fill(index + 11, 0.4)); - problem.general_node_response = {1, 2, 0, 1, 2}; + problem.general.node_response = {1, 2, 0, 1, 2}; } return problem; } std::vector scaled_state(const native::Problem& problem) { std::vector state; - for (std::size_t index = 0; index < kModes * problem.n_omega; ++index) + state.reserve(kModes * problem.dims.n_omega); + for (std::size_t index = 0; index < kModes * problem.dims.n_omega; ++index) state.emplace_back(fill(index + 17, 1.0), fill(index + 23, 1.0)); return state; } @@ -131,8 +131,8 @@ std::vector scaled_state(const native::Problem& problem) { // solprop's `general_source`, written as sums over samples and bins. std::vector reference_source(const native::Problem& problem, const std::vector& scaled) { - const auto n_to = problem.n_to; - const auto n_omega = problem.n_omega; + const auto n_to = problem.dims.n_to; + const auto n_omega = problem.dims.n_omega; const auto components = kComponents; const auto two_pi = 2.0 * std::numbers::pi_v; std::vector source(kModes * n_omega); @@ -142,19 +142,17 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t w = 0; w < n_omega; ++w) { Complex bin{}; for (std::size_t m = 0; m < kModes; ++m) - bin += - problem - .general_synthesis[((node * components + c) * kModes + m) * n_omega + - w] * - scaled[m * n_omega + w]; - const auto fine = problem.crop_index[w]; + bin += problem.general + .synthesis[((node * components + c) * kModes + m) * n_omega + w] * + scaled[m * n_omega + w]; + const auto fine = problem.grid.crop_index[w]; for (std::size_t t = 0; t < n_to; ++t) field[c][t] += bin * cis(two_pi * static_cast(fine * t) / static_cast(n_to)) / static_cast(n_to); } - if (problem.general_real_form) + if (problem.general.real_form) for (auto& component : field) for (auto& value : component) value = Complex(value.real(), 0.0); @@ -167,17 +165,18 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t j = 0; j < components; ++j) for (std::size_t k = 0; k < components; ++k) for (std::size_t l = 0; l < components; ++l) - value += problem.general_tensor[base + - ((i * components + j) * components + k) * - components + - l] * - field[k][t] * Kokkos::conj(field[l][t]) * field[j][t]; - polarisation[i][t] = problem.general_epsilon0 * value.real(); + value += + problem.general + .tensor[base + + ((i * components + j) * components + k) * components + + l] * + field[k][t] * Kokkos::conj(field[l][t]) * field[j][t]; + polarisation[i][t] = problem.general.epsilon0 * value.real(); } - if (problem.general_terms != 0 && problem.general_node_response[node] != 0) { - const auto owner = problem.general_node_response[node] - 1; + if (problem.general.terms != 0 && problem.general.node_response[node] != 0) { + const auto owner = problem.general.node_response[node] - 1; const auto doubled = 2 * n_to; - for (std::size_t term = 0; term < problem.general_terms; ++term) { + for (std::size_t term = 0; term < problem.general.terms; ++term) { std::vector>> history( components, std::vector>(components, std::vector(n_to))); @@ -186,7 +185,7 @@ std::vector reference_source(const native::Problem& problem, std::vector drive(doubled); for (std::size_t t = 0; t < n_to; ++t) { const double pair = (field[k][t] * Kokkos::conj(field[l][t])).real(); - drive[t] = problem.general_real_form ? pair : 0.5 * pair; + drive[t] = problem.general.real_form ? pair : 0.5 * pair; } std::vector product(n_to + 1); for (std::size_t f = 0; f <= n_to; ++f) { @@ -195,8 +194,8 @@ std::vector reference_source(const native::Problem& problem, bin += drive[t] * cis(-two_pi * static_cast(f * t) / static_cast(doubled)); product[f] = - problem.general_delayed_spectrum - [(owner * problem.general_terms + term) * (n_to + 1) + f] * + problem.general.delayed_spectrum + [(owner * problem.general.terms + term) * (n_to + 1) + f] * bin; } for (std::size_t t = 0; t < n_to; ++t) { @@ -209,7 +208,7 @@ std::vector reference_source(const native::Problem& problem, history[k][l][t] = value / static_cast(doubled); } } - const auto tensor = (owner * problem.general_terms + term) * fourth(components); + const auto tensor = (owner * problem.general.terms + term) * fourth(components); for (std::size_t t = 0; t < n_to; ++t) for (std::size_t i = 0; i < components; ++i) { double value = 0.0; @@ -217,32 +216,31 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t k = 0; k < components; ++k) for (std::size_t l = 0; l < components; ++l) value += - problem.general_delayed_tensor + problem.general.delayed_tensor [tensor + ((i * components + j) * components + k) * components + l] * history[k][l][t] * field[j][t].real(); - polarisation[i][t] += problem.general_epsilon0 * value; + polarisation[i][t] += problem.general.epsilon0 * value; } } } for (std::size_t c = 0; c < components; ++c) for (std::size_t w = 0; w < n_omega; ++w) { - const auto fine = problem.crop_index[w]; + const auto fine = problem.grid.crop_index[w]; Complex bin{}; for (std::size_t t = 0; t < n_to; ++t) bin += polarisation[c][t] * cis(-two_pi * static_cast(fine * t) / static_cast(n_to)); for (std::size_t m = 0; m < kModes; ++m) source[m * n_omega + w] += - problem - .general_projection[((m * kNodes + node) * components + c) * n_omega + - w] * + problem.general + .projection[((m * kNodes + node) * components + c) * n_omega + w] * bin; } } for (std::size_t m = 0; m < kModes; ++m) for (std::size_t w = 0; w < n_omega; ++w) - source[m * n_omega + w] *= problem.general_frequency_factor[w]; + source[m * n_omega + w] *= problem.general.frequency_factor[w]; return source; } @@ -252,15 +250,15 @@ std::vector evaluated(const native::Problem& problem, source.plan(); auto input = Kokkos::create_mirror_view(source.scaled()); for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); source.evaluate(); const auto output = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, source.source()); std::vector result; for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) result.push_back(output(m, w)); return result; } @@ -377,8 +375,8 @@ TEST(GeneralModal, RepeatedEvaluationAgreesToRounding) { auto input = Kokkos::create_mirror(source.scaled()); for (int repeat = 0; repeat < 3; ++repeat) { for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); source.evaluate(); const auto output = @@ -400,31 +398,31 @@ TEST(GeneralModal, RepeatedEvaluationAgreesToRounding) { TEST(GeneralModal, MalformedDataIsRefusedBeforeAllocation) { auto short_map = modal_problem(true, 2, false); - short_map.general_synthesis.pop_back(); + short_map.general.synthesis.pop_back(); EXPECT_THROW(native::require_general_modal(short_map), std::invalid_argument); auto chunk = modal_problem(true, 2, false); - chunk.general_node_chunk = kNodes + 1; + chunk.general.node_chunk = kNodes + 1; EXPECT_THROW(native::require_general_modal(chunk), std::invalid_argument); auto owner = modal_problem(true, 2, true); - owner.general_node_response[0] = kResponses + 1; + owner.general.node_response[0] = kResponses + 1; EXPECT_THROW(native::require_general_modal(owner), std::invalid_argument); auto two = modal_problem(true, 2, true); - two.general_components = 2; + two.general.components = 2; EXPECT_THROW(native::require_general_modal(two), std::invalid_argument); auto legacy = modal_problem(true, 2, false); - legacy.nonlinear = true; + legacy.kerr.nonlinear = true; EXPECT_THROW(native::require_general_modal(legacy), std::invalid_argument); auto envelope = modal_problem(true, 2, false); - envelope.family = native::FieldFamily::envelope; + envelope.dims.family = native::FieldFamily::envelope; EXPECT_THROW(native::require_general_modal(envelope), std::invalid_argument); auto scattered = modal_problem(true, 2, false); - scattered.crop_index = {0, 2, 1}; + scattered.grid.crop_index = {0, 2, 1}; EXPECT_THROW(native::require_general_modal(scattered), std::invalid_argument); auto stray = modal_problem(true, 2, false); - stray.general_modal = false; + stray.general.enabled = false; EXPECT_THROW(native::require_general_modal(stray), std::invalid_argument); auto infinite = modal_problem(true, 2, false); - infinite.general_tensor[3] = std::numeric_limits::infinity(); + infinite.general.tensor[3] = std::numeric_limits::infinity(); EXPECT_THROW(native::GeneralState state(infinite), std::invalid_argument); } @@ -434,8 +432,8 @@ TEST(GeneralModal, FullAndTailTransformsUseTheDedicatedPolicy) { auto problem = modal_problem(true, 3, false, 4096, 0, 16384); native::GeneralModalSource source(problem); const auto sizing = source.execution_sizing(); - const auto full = 3 * kComponents * kModes * problem.n_omega; - const auto tail = 2 * kComponents * kModes * problem.n_omega; + const auto full = 3 * kComponents * kModes * problem.dims.n_omega; + const auto tail = 2 * kComponents * kModes * problem.dims.n_omega; const auto capacity = static_cast(native::ExecutionSpace().concurrency()); EXPECT_EQ(sizing.at("kernels.modal." + std::to_string(full)).at("threads"), capacity); @@ -457,11 +455,11 @@ TEST(GeneralModal, LedgerRowsAreTheAllocatedViews) { const auto real = sizeof(double); const auto rows = 2 * kComponents; const auto pairs = 2 * native::general_pair_histories; - const auto n_to = problem.n_to; - EXPECT_EQ(ledger.at("general_synthesis"), problem.general_synthesis.size() * complex); + const auto n_to = problem.dims.n_to; + EXPECT_EQ(ledger.at("general_synthesis"), problem.general.synthesis.size() * complex); EXPECT_EQ(ledger.at("general_projection"), - problem.general_projection.size() * complex); - EXPECT_EQ(ledger.at("general_tensor"), problem.general_tensor.size() * real); + problem.general.projection.size() * complex); + EXPECT_EQ(ledger.at("general_tensor"), problem.general.tensor.size() * real); EXPECT_EQ(ledger.at("general_node_spectrum"), rows * n_to * complex); EXPECT_EQ(ledger.at("general_node_field"), rows * n_to * complex); EXPECT_EQ(ledger.at("general_node_polarisation"), rows * n_to * real); @@ -472,7 +470,7 @@ TEST(GeneralModal, LedgerRowsAreTheAllocatedViews) { EXPECT_EQ(ledger.at("general_node_product"), pairs * (n_to + 1) * complex); EXPECT_EQ(ledger.at("general_node_delayed"), rows * n_to * real); EXPECT_EQ(ledger.at("general_delayed_spectrum"), - problem.general_delayed_spectrum.size() * complex); + problem.general.delayed_spectrum.size() * complex); EXPECT_EQ(ledger.at("general_node_response"), kNodes * sizeof(std::size_t)); } @@ -481,21 +479,34 @@ TEST(GeneralModal, TheStateCarriesTheSourceAndItsLedger) { native::GeneralState state(problem); const auto ledger = state.memory_ledger(); EXPECT_EQ(ledger.at("general_synthesis"), - problem.general_synthesis.size() * sizeof(Complex)); + problem.general.synthesis.size() * sizeof(Complex)); // The host maps are released once uploaded; the retained problem is small. const auto host_maps = - (problem.general_synthesis.size() + problem.general_projection.size() + - problem.general_delayed_spectrum.size()) * + (problem.general.synthesis.size() + problem.general.projection.size() + + problem.general.delayed_spectrum.size()) * sizeof(Complex) + - (problem.general_tensor.size() + problem.general_delayed_tensor.size()) * + (problem.general.tensor.size() + problem.general.delayed_tensor.size()) * sizeof(double); EXPECT_LT(ledger.at("host_problem_arrays"), host_maps); - std::vector field(kModes * problem.n_omega); + std::vector field(kModes * problem.dims.n_omega); for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); - const native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + const native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; const auto first = state.trial_step(0.01, scalars); const auto stage = state.stage_to_host(0); const double magnitude = std::accumulate( @@ -514,7 +525,7 @@ TEST(GeneralModal, TheStateCarriesTheSourceAndItsLedger) { TEST(GeneralModal, AnEdgeLineReadsAPlainTransformAndASlope) { const auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - const auto n = problem.n_omega; + const auto n = problem.dims.n_omega; // Rows flowing both ways, so that rectifying a sum would read something else. std::vector field(kModes * n); for (std::size_t row = 0; row < n; ++row) { @@ -615,16 +626,16 @@ TEST(GeneralModal, AnEdgeLineThatDoesNotAccountForItsNodesIsRefused) { TEST(GeneralModal, TheAdamsRowsRoundTripExactly) { auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); constexpr std::size_t kSlots = 4; state.adams_allocate(kSlots, true, false); EXPECT_EQ(state.adams_row_names(), (std::vector{"store", "states", "rates"})); - const std::array extents{kSlots, kModes, problem.n_omega}; + const std::array extents{kSlots, kModes, problem.dims.n_omega}; for (const std::string name : {"store", "states", "rates"}) { EXPECT_EQ(state.adams_row_extents(name), extents); - std::vector rows(kSlots * kModes * problem.n_omega); + std::vector rows(kSlots * kModes * problem.dims.n_omega); for (std::size_t index = 0; index < rows.size(); ++index) rows[index] = Complex(fill(index, 1.0), fill(index + name.size(), 1e-300)); state.adams_rows_from_host(name, rows); @@ -657,7 +668,7 @@ TEST(GeneralModal, TheAdamsRowsRoundTripExactly) { TEST(GeneralModal, TheAdamsRowsNeedTheHistory) { auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); EXPECT_THROW(static_cast(state.adams_row_names()), std::logic_error); state.adams_allocate(3, false, false); @@ -667,12 +678,25 @@ TEST(GeneralModal, TheAdamsRowsNeedTheHistory) { TEST(GeneralModal, ALinearGuideLeavesTheStageZero) { auto problem = modal_problem(true, 2, false); - std::fill(problem.general_tensor.begin(), problem.general_tensor.end(), 0.0); + std::ranges::fill(problem.general.tensor, 0.0); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); - const native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + const native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; static_cast(state.trial_step(0.01, scalars)); // Exact: a zero tensor contracts to exact zeros. for (const auto value : state.stage_to_host(0)) { @@ -700,23 +724,23 @@ constexpr double kCombinationBound = 2 * kDegree * kEpsilon; // other terms fall off as a Chebyshev series' do. native::Problem moving_problem(const bool real_form, const std::size_t stages = 2) { auto problem = modal_problem(real_form, 2, false); - problem.n_stages = stages; - problem.general_map_intervals = kSections; - problem.general_map_n_deg = kDegree; - const auto synthesis = problem.general_synthesis; - const auto projection = problem.general_projection; + problem.stepper.n_stages = stages; + problem.general.map_intervals = kSections; + problem.general.map_n_deg = kDegree; + const auto synthesis = problem.general.synthesis; + const auto projection = problem.general.projection; const auto maps = synthesis.size(); - problem.general_synthesis.clear(); - problem.general_projection.clear(); + problem.general.synthesis.clear(); + problem.general.projection.clear(); for (std::size_t term = 0; term < kMapTerms; ++term) { const double scale = 1.0 / static_cast(1 + term % kDegree); for (std::size_t index = 0; index < maps; ++index) { const auto shift = 31 * term; - problem.general_synthesis.push_back( + problem.general.synthesis.push_back( term == 0 ? synthesis[index] : Complex(fill(index + shift, scale), fill(index + shift + 5, scale))); - problem.general_projection.push_back( + problem.general.projection.push_back( term == 0 ? projection[index] : Complex(fill(index + shift + 9, scale), fill(index + shift + 2, scale))); @@ -763,8 +787,8 @@ void load_scaled(native::GeneralModalSource& source, const native::Problem& prob const std::vector& scaled) { auto input = Kokkos::create_mirror_view(source.scaled()); for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); } @@ -778,7 +802,7 @@ TEST(GeneralModal, MovingMapsAreEachStageSWeightedSumOfItsSectionSTerms) { source.combine(stage); for (const bool synthesis : {true, false}) { const auto& terms = - synthesis ? problem.general_synthesis : problem.general_projection; + synthesis ? problem.general.synthesis : problem.general.projection; const auto actual = to_host(synthesis ? source.synthesis() : source.projection()); const auto expected = host_map(terms, weights, stage, false); const auto magnitude = host_map(terms, weights, stage, true); @@ -809,7 +833,7 @@ TEST(GeneralModal, OneSelectedTermIsTheConstantGuideOfThatTerm) { source.combine(0); for (const bool synthesis : {true, false}) { const auto& terms = - synthesis ? problem.general_synthesis : problem.general_projection; + synthesis ? problem.general.synthesis : problem.general.projection; const auto maps = to_host(synthesis ? source.synthesis() : source.projection()); const auto selected = host_map(terms, weights, 0, false); ASSERT_EQ(maps.size(), selected.size()); @@ -833,10 +857,10 @@ TEST(GeneralModal, AMovingStageSSourceIsTheSourceOfItsCombinedMaps) { const auto weights = moving_weights(); for (std::size_t stage = 0; stage < 2; ++stage) { auto combined = modal_problem(false, 2, false); - combined.general_synthesis = - host_map(problem.general_synthesis, weights, stage, false); - combined.general_projection = - host_map(problem.general_projection, weights, stage, false); + combined.general.synthesis = + host_map(problem.general.synthesis, weights, stage, false); + combined.general.projection = + host_map(problem.general.projection, weights, stage, false); const auto scaled = scaled_state(combined); native::GeneralModalSource source(problem); source.plan(); @@ -877,13 +901,13 @@ TEST(GeneralModal, MovingMapsRefuseMalformedWeightsAndAMissingTrial) { fixed.set_weights({}); fixed.combine(0); auto short_terms = moving_problem(true); - short_terms.general_projection.resize(short_terms.general_projection.size() - 1); + short_terms.general.projection.resize(short_terms.general.projection.size() - 1); EXPECT_THROW(native::require_general_modal(short_terms), std::invalid_argument); auto no_sections = moving_problem(true); - no_sections.general_map_intervals = 0; + no_sections.general.map_intervals = 0; EXPECT_THROW(native::require_general_modal(no_sections), std::invalid_argument); auto no_stages = moving_problem(true); - no_stages.n_stages = 0; + no_stages.stepper.n_stages = 0; EXPECT_THROW(native::require_general_modal(no_stages), std::invalid_argument); } @@ -892,7 +916,7 @@ TEST(GeneralModal, MovingLedgerRowsAreTheTermsTheStageMapsAndTheWeights) { native::GeneralModalSource source(problem); const auto ledger = source.memory_ledger(); const auto complex = sizeof(Complex); - const auto one = problem.general_synthesis.size() / kMapTerms; + const auto one = problem.general.synthesis.size() / kMapTerms; EXPECT_EQ(ledger.at("general_synthesis"), kMapTerms * one * complex); EXPECT_EQ(ledger.at("general_projection"), kMapTerms * one * complex); EXPECT_EQ(ledger.at("general_stage_maps"), 2 * one * complex); @@ -909,13 +933,26 @@ TEST(GeneralModal, TheStateCombinesAMovingGuideSMapsPerTrial) { const auto problem = moving_problem(true, 1); native::GeneralState state(problem); EXPECT_EQ(state.memory_ledger().at("general_stage_maps"), - 2 * problem.general_synthesis.size() / kMapTerms * sizeof(Complex)); - std::vector field(kModes * problem.n_omega); + 2 * problem.general.synthesis.size() / kMapTerms * sizeof(Complex)); + std::vector field(kModes * problem.dims.n_omega); for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); - native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; EXPECT_THROW(static_cast(state.trial_step(0.01, scalars)), std::invalid_argument); scalars.transform_weights.assign(kMapTerms, 0.0); diff --git a/native/tests/native/test_helpers.cpp b/native/tests/native/test_helpers.cpp index 6ca2b0787..f9a012cee 100644 --- a/native/tests/native/test_helpers.cpp +++ b/native/tests/native/test_helpers.cpp @@ -159,7 +159,7 @@ TEST(LagrangeBasis, TrialTablesOnEqualSteps) { EXPECT_EQ(corrector[1], 0.5); EXPECT_EQ(corrector[2], 0.5); for (std::size_t column = 0; column < 4; ++column) { - EXPECT_EQ(corrector[3 * 4 + column], 0.0); // past its nodes + EXPECT_EQ(corrector[std::size_t{3} * 4 + column], 0.0); // past its nodes EXPECT_EQ(corrector[column * 4 + 3], 0.0); } const std::array first_order{1.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0}; diff --git a/native/tests/native/test_low_storage.cpp b/native/tests/native/test_low_storage.cpp index 5a91718d9..302386208 100644 --- a/native/tests/native/test_low_storage.cpp +++ b/native/tests/native/test_low_storage.cpp @@ -10,6 +10,9 @@ // flight, by a failure or a divergence. #include "solprop_native/low_storage.hpp" +#include "solprop_native/session.hpp" + +#include #include #include @@ -19,10 +22,6 @@ #include #include -#include - -#include "solprop_native/session.hpp" - namespace { namespace low_storage = solprop_native::rk::low_storage; namespace march = solprop_native::march; @@ -134,7 +133,7 @@ TEST(LowStorage, EachPositionIsItsOwnAnchorAndTheEndPadsTheStages) { const double dz = 0.0625; const double midpoint = z + 0.5 * dz; const double end = z + dz; - const low_storage::Config recompute{0, false, 0.0}; + const low_storage::Config recompute{.flows = 0, .separable = false, .origin = 0.0}; const auto scalars = low_storage::scalars(tables, anchor, recompute, z, dz, false, std::nan("")); EXPECT_EQ(scalars.phi_at, (std::vector{z, midpoint, end, end})); @@ -154,7 +153,7 @@ TEST(LowStorage, AConstantOperatorsTableIsBuiltFromItsOrigin) { const double z = 0.375; const double dz = 0.1; const double key = 0.09999999999999998; - const low_storage::Config table{1, false, 0.25}; + const low_storage::Config table{.flows = 1, .separable = false, .origin = 0.25}; const auto built = low_storage::scalars(tables, anchor, table, z, dz, true, key); // fl(origin + key / 2) - origin, not key / 2: solprop's engines build it so. const double middle = 0.25 + 0.5 * key; @@ -162,7 +161,7 @@ TEST(LowStorage, AConstantOperatorsTableIsBuiltFromItsOrigin) { const auto reused = low_storage::scalars(tables, anchor, table, z, dz, false, key); EXPECT_EQ(reused.phi_delta, (std::vector(5, 0.0))); // A separable operator's increments are the distances themselves. - const low_storage::Config separable{1, true, 0.25}; + const low_storage::Config separable{.flows = 1, .separable = true, .origin = 0.25}; const auto spatial = low_storage::scalars(tables, anchor, separable, z, dz, true, key); EXPECT_EQ(spatial.phi_delta, @@ -188,7 +187,9 @@ TEST(LowStorage, AnExpansionsIncrementsAreItsWeightsDifferences) { const double dz = 0.2; const double midpoint = z + 0.5 * dz; const auto scalars = low_storage::scalars( - tables, anchor, low_storage::Config{2, false, 0.0}, z, dz, false, std::nan("")); + tables, anchor, + low_storage::Config{.flows = 2, .separable = false, .origin = 0.0}, z, dz, false, + std::nan("")); stage::Anchor fresh; const auto first = stage::row(tables, fresh, z, midpoint).phi_delta; const auto second = stage::row(tables, fresh, midpoint, z + dz).phi_delta; diff --git a/native/tests/native/test_rk_march.cpp b/native/tests/native/test_rk_march.cpp index 937b2070f..e5ce98eea 100644 --- a/native/tests/native/test_rk_march.cpp +++ b/native/tests/native/test_rk_march.cpp @@ -8,12 +8,12 @@ #include "solprop_native/rk_march.hpp" +#include + #include #include #include -#include - namespace { namespace rk = solprop_native::rk; namespace march = solprop_native::march; diff --git a/native/tests/native/test_session.cpp b/native/tests/native/test_session.cpp index 4d89b162a..66de519ba 100644 --- a/native/tests/native/test_session.cpp +++ b/native/tests/native/test_session.cpp @@ -11,6 +11,8 @@ #include "solprop_native/session.hpp" +#include + #include #include #include @@ -23,8 +25,6 @@ #include #include -#include - namespace { namespace session = solprop_native::session; namespace march = solprop_native::march; @@ -78,6 +78,7 @@ class ToyDense final : public session::Dense { void stepped(const march::Result& /*result*/) override {} [[nodiscard]] bool crosses(const double position, const double target, const int ulps) const override { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) return !march::reached(target, position, ulps); } void observe(const double position) override { @@ -258,17 +259,17 @@ std::vector drive(session::Session& run, const std::size_t work_limit, run.acknowledge(batch.sequence); continue; } - Seen entry{event.kind, - event.observation, - event.z, - event.plane_index, - std::move(held.attempts), - std::move(held.row_z), - std::move(held.row_step), - std::move(held.rows), - event.has_stats ? batch.plane_stats : std::vector{}, - {}, - event.outcome}; + Seen entry{.kind = event.kind, + .observation = event.observation, + .z = event.z, + .plane = event.plane_index, + .attempts = std::move(held.attempts), + .row_z = std::move(held.row_z), + .row_step = std::move(held.row_step), + .rows = std::move(held.rows), + .stats = event.has_stats ? batch.plane_stats : std::vector{}, + .field = {}, + .outcome = event.outcome}; held = Held{}; if (event.ticket) { entry.field.resize(2 * run.elements()); @@ -491,7 +492,7 @@ TEST(Session, StatusAndControlsNeedNoLock) { std::atomic gate{false}; built.engine->hold(&entered, &gate); session::Batch batch; - std::thread advancing([&]() { batch = run.advance(256, 10.0); }); + std::thread advancing([&] { batch = run.advance(256, 10.0); }); while (!entered.load()) std::this_thread::yield(); const auto status = run.status(); diff --git a/native/tests/native/test_session_monitor.cpp b/native/tests/native/test_session_monitor.cpp index a5479f5c7..6f222d670 100644 --- a/native/tests/native/test_session_monitor.cpp +++ b/native/tests/native/test_session_monitor.cpp @@ -13,14 +13,14 @@ #include "solprop_native/session.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace session = solprop_native::session; namespace rk = solprop_native::rk; @@ -149,6 +149,7 @@ session::Batch next(session::Session& run, const session::EventKind kind, // The rows' accepted-step values a batch carries. std::vector rows_of(const session::Batch& batch) { std::vector steps; + steps.reserve(batch.row_z.size()); for (std::size_t row = 0; row < batch.row_z.size(); ++row) steps.push_back(batch.row_values[2 * row + 1]); return steps; @@ -193,8 +194,8 @@ TEST(SessionMonitor, AWarningIsABarrierBeforeItsStepsRow) { ASSERT_FALSE(rows.empty()); for (std::size_t index = 0; index < rows.size(); ++index) EXPECT_EQ(rows[index], 3.0 + static_cast(index)); - EXPECT_EQ(rows.back(), - static_cast(3 + plane.attempts.size() / session::attempt_columns)); + const std::size_t attempts = plane.attempts.size() / session::attempt_columns; + EXPECT_EQ(rows.back(), static_cast(3 + attempts)); next(run, session::EventKind::observation); next(run, session::EventKind::terminal); const auto record = run.monitor_record(); diff --git a/native/tests/native/test_session_transverse.cpp b/native/tests/native/test_session_transverse.cpp index 47d466f54..4d41f40b0 100644 --- a/native/tests/native/test_session_transverse.cpp +++ b/native/tests/native/test_session_transverse.cpp @@ -14,6 +14,8 @@ #include "solprop_native/session.hpp" +#include + #include #include #include @@ -21,8 +23,6 @@ #include #include -#include - namespace { namespace session = solprop_native::session; namespace rk = solprop_native::rk; diff --git a/native/tests/native/test_transverse.cpp b/native/tests/native/test_transverse.cpp index 578c56b2b..5b5afd167 100644 --- a/native/tests/native/test_transverse.cpp +++ b/native/tests/native/test_transverse.cpp @@ -11,14 +11,14 @@ #include "solprop_native/transverse.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace transverse = solprop_native::transverse; using Kind = transverse::Notice::Kind; diff --git a/native/tests/native/test_vmath.cpp b/native/tests/native/test_vmath.cpp index 4e12931ef..c428d79c1 100644 --- a/native/tests/native/test_vmath.cpp +++ b/native/tests/native/test_vmath.cpp @@ -61,7 +61,12 @@ TEST(VectorMath, EveryElementOfABlockIsItsOwnValue) { for (const std::size_t n : lengths()) { const auto x = uniform(n, -30.0, 5.0, 11U + static_cast(n)); const auto angle = uniform(n, -400.0, 400.0, 29U + static_cast(n)); - std::vector e(n), l(n), s(n), c(n), re(n), im(n); + std::vector e(n); + std::vector l(n); + std::vector s(n); + std::vector c(n); + std::vector re(n); + std::vector im(n); std::vector positive(n); for (std::size_t i = 0; i < n; ++i) positive[i] = std::exp(x[i]); @@ -70,8 +75,12 @@ TEST(VectorMath, EveryElementOfABlockIsItsOwnValue) { vmath::sincos(angle.data(), s.data(), c.data(), n); vmath::exp_complex(x.data(), angle.data(), re.data(), im.data(), n); for (std::size_t i = 0; i < n; ++i) { - double one_e = 0.0, one_l = 0.0, one_s = 0.0, one_c = 0.0, one_re = 0.0, - one_im = 0.0; + double one_e = 0.0; + double one_l = 0.0; + double one_s = 0.0; + double one_c = 0.0; + double one_re = 0.0; + double one_im = 0.0; vmath::exp(&x[i], &one_e, 1); vmath::log(&positive[i], &one_l, 1); vmath::sincos(&angle[i], &one_s, &one_c, 1); @@ -94,7 +103,8 @@ TEST(VectorMath, InPlaceGivesTheSameValues) { auto in_place = x; vmath::exp(in_place.data(), in_place.data(), n); EXPECT_EQ(in_place, out); - std::vector s(n), c(n); + std::vector s(n); + std::vector c(n); vmath::sincos(x.data(), s.data(), c.data(), n); auto sine = x; vmath::sincos(sine.data(), sine.data(), c.data(), n); @@ -112,7 +122,12 @@ TEST(VectorMath, AgreesWithTheSystemLibm) { const auto magnitude_exponent = uniform(n, -300.0, 300.0, 13U); const auto phase = uniform(n, -1.0e4, 1.0e4, 17U); const auto huge_phase = uniform(n, -1.0e15, 1.0e15, 19U); - std::vector out(n), s(n), c(n), re(n), im(n), magnitude(n); + std::vector out(n); + std::vector s(n); + std::vector c(n); + std::vector re(n); + std::vector im(n); + std::vector magnitude(n); for (const auto* values : {&exponent, &small}) { vmath::exp(values->data(), out.data(), n); for (std::size_t i = 0; i < n; ++i) @@ -179,7 +194,8 @@ TEST(VectorMath, SpecialValuesFollowC99) { expect_within_two_ulps(l[6], std::log(std::numeric_limits::denorm_min())); const std::vector angle = {0.0, -0.0, infinity, -infinity, quiet_nan}; - std::vector s(angle.size()), c(angle.size()); + std::vector s(angle.size()); + std::vector c(angle.size()); vmath::sincos(angle.data(), s.data(), c.data(), angle.size()); EXPECT_EQ(s[0], 0.0); EXPECT_FALSE(std::signbit(s[0])); @@ -196,10 +212,12 @@ TEST(VectorMath, SpecialValuesFollowC99) { // NaN in either part makes both parts NaN. const std::vector a = {0.0, -0.0, quiet_nan, 0.0}; const std::vector b = {0.5, -0.5, 0.5, quiet_nan}; - std::vector re(a.size()), im(a.size()); + std::vector re(a.size()); + std::vector im(a.size()); vmath::exp_complex(a.data(), b.data(), re.data(), im.data(), a.size()); for (std::size_t i = 0; i < 2; ++i) { - double sine = 0.0, cosine = 0.0; + double sine = 0.0; + double cosine = 0.0; vmath::sincos(&b[i], &sine, &cosine, 1); EXPECT_EQ(re[i], cosine); EXPECT_EQ(im[i], sine); @@ -211,7 +229,7 @@ TEST(VectorMath, SpecialValuesFollowC99) { } TEST(VectorMath, RouteMatchesTheBuild) { -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF EXPECT_TRUE(vmath::vectorised); EXPECT_GE(vmath::width, 2U); #else diff --git a/native/tests/native/test_watch.cpp b/native/tests/native/test_watch.cpp index 92bb6916f..5bffa8e72 100644 --- a/native/tests/native/test_watch.cpp +++ b/native/tests/native/test_watch.cpp @@ -9,14 +9,14 @@ #include "solprop_native/watch.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace watch = solprop_native::watch; diff --git a/native/tests/native/test_z_table.cpp b/native/tests/native/test_z_table.cpp index 324e8bb5d..7bd5ad8d2 100644 --- a/native/tests/native/test_z_table.cpp +++ b/native/tests/native/test_z_table.cpp @@ -4,6 +4,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/stage_tables.hpp" #include "solprop_native/z_table.hpp" #include @@ -15,6 +16,7 @@ #include #include #include +#include #include namespace { @@ -199,4 +201,31 @@ TEST(ZTable, BlockedEvaluationIsThePerValueRecursionExactly) { } } +// A scalar stage table holds one value: the loader refuses one of two by name, before +// any read could write past the single value it returns. +TEST(StageTable, LoaderRefusesAMultiValuedScalarTableByName) { + native::stage::Table table; + table.edges = {0.0, 1.0}; + table.lower = 0.0; + table.upper = 1.0; + table.n_terms = 1; + table.n_values = 1; + table.coefficients = {2.5}; + native::stage::Tables tables; + tables.density = table; + EXPECT_NO_THROW(native::stage::require_scalar_tables(tables)); + EXPECT_EQ(tables.density.value(0.5), 2.5); + auto wide = table; + wide.n_values = 2; + wide.coefficients = {2.5, -1.0}; + tables.plasma_densities = {table, wide}; + try { + native::stage::require_scalar_tables(tables); + FAIL() << "a two-value plasma density table was accepted"; + } catch (const std::invalid_argument& error) { + EXPECT_NE(std::string(error.what()).find("plasma_densities[1]"), std::string::npos) + << error.what(); + } +} + } // namespace diff --git a/native/tests/test_api_contract.py b/native/tests/test_api_contract.py index bb973914d..2b4399f7b 100644 --- a/native/tests/test_api_contract.py +++ b/native/tests/test_api_contract.py @@ -103,12 +103,7 @@ def test_build_info_is_frozen_and_complete() -> None: assert info.kokkos_version == "5.2.1" assert info.kokkos_fft_version == "2.0.0" assert info.kokkos_kernels_version == "5.2.0" - assert info.gemm_provider in { - "CUSTOM", - "KOKKOSBLAS_NATIVE", - "KOKKOSBLAS_TPL", - "HYBRID", - } + assert info.gemm_provider == "HYBRID" if info.tpl_blas_enabled: assert info.blas_backend != "none" else: @@ -169,13 +164,9 @@ def test_build_info_is_frozen_and_complete() -> None: assert info.cpu_baseline in info.compiler_flags assert info.cpu_baseline in info.optimization_policy if info.device_backend == "none": - if info.openmp_enabled: - assert info.execution_space == "OpenMP" - assert info.fft_backend in {"FFTW_OPENMP", "FFTW_PTHREADS", "MKL_FFTW"} - else: - # The Kokkos Threads build adds no OpenMP runtime (solprop SEP 29). - assert info.execution_space == "Threads" - assert info.fft_backend == "FFTW_THREADS" + assert info.openmp_enabled + assert info.execution_space == "OpenMP" + assert info.fft_backend in {"FFTW_OPENMP", "FFTW_PTHREADS", "MKL_FFTW"} for count in ( info.thread_count, info.fftw_threads, @@ -727,15 +718,6 @@ def run() -> ComplexArray: def run_ignored(_: int) -> ComplexArray: return run() - if native.build_info().execution_space == "Threads": - # Kokkos Threads dispatches only from the thread that started it: another - # Python thread gets a RuntimeError, never a crash (solprop SEP 29 phase 0). - with ( - ThreadPoolExecutor(max_workers=1) as executor, - pytest.raises(RuntimeError, match="master"), - ): - executor.submit(run).result() - return with ThreadPoolExecutor(max_workers=2) as executor: results = list(executor.map(run_ignored, range(2))) bound = step_bound(grid.n_to, 4) diff --git a/native/tests/test_backend.py b/native/tests/test_backend.py index 3352168ff..4e0a33154 100644 --- a/native/tests/test_backend.py +++ b/native/tests/test_backend.py @@ -45,7 +45,8 @@ import solprop_native from solprop_native import NativeBackend, _core, build_info, check_build from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] _ResidentLowStorage, # pyright: ignore[reportPrivateUsage] ) from solprop_native.session import NativeSession @@ -106,7 +107,7 @@ def test_native_modal_payload_requires_both_fixed_coefficient_arrays( prefactor=None if missing == "prefactor" else terms.prefactor, ) with pytest.raises(BackendError, match="device-ready fixed-term problem"): - _native_problem(problem, run_request, damaged) + _lower(problem, run_request, _LowerPlan(terms=damaged)) finally: backend.close() @@ -646,14 +647,14 @@ def test_raman_metadata_memoization_and_conditional_ledger( backend.close() -def _raman_native_payload(raman_case: RamanCase) -> dict[str, object]: +def _raman_native_payload(raman_case: RamanCase) -> _core.Problem: problem, run_request, grid = raman_case backend = NativeBackend(problem, run_request, grid=grid) try: - return _native_problem( # pyright: ignore[reportPrivateUsage] + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] ) finally: backend.close() @@ -665,25 +666,30 @@ def test_native_raman_rows_refuse_hidden_array_conversions( ) -> None: """The private seam accepts exact arrays and never copies a near miss.""" payload = _raman_native_payload(raman_case) - rows = cast(np.ndarray, payload["raman_undamped"]) - if spoiling == "dtype": - payload["raman_undamped"] = rows.astype(np.float32) - elif spoiling == "rank": - payload["raman_undamped"] = rows.ravel() - else: - payload["raman_undamped"] = np.zeros( - (rows.shape[0], 2 * rows.shape[1]), dtype=np.float64 - )[:, ::2] - with pytest.raises((TypeError, ValueError)): + raman = payload.raman + rows = raman.undamped.reshape(raman.n_terms, -1) + + def _spoil() -> None: + if spoiling == "dtype": + raman.undamped = rows.astype(np.float32) # pyright: ignore[reportAttributeAccessIssue] + elif spoiling == "rank": + raman.undamped = rows.ravel() + else: + raman.undamped = np.zeros( + (rows.shape[0], 2 * rows.shape[1]), dtype=np.float64 + )[:, ::2] _core.State(payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + def test_native_raman_rows_accept_read_only_input(raman_case: RamanCase) -> None: """Construction reads Raman rows but does not require write access to them.""" payload = _raman_native_payload(raman_case) - rows = np.array(cast(np.ndarray, payload["raman_undamped"]), copy=True) + rows = payload.raman.undamped.reshape(payload.raman.n_terms, -1) rows.setflags(write=False) - payload["raman_undamped"] = rows + payload.raman.undamped = rows state = _core.State(payload) state.close() @@ -723,21 +729,10 @@ def test_metadata_and_memory_ledger_are_self_consistent( ).endswith(("thread", "threads")) assert cast(float, metadata["planning_seconds"]) >= 0.0 # A one-mode envelope at one node. A host build folds its one-element transforms - # into the response column (solprop SEP 25), whatever the GEMM policy. A device - # build's forced vendor policy multiplies its real transform matrices into the - # samples' interleaved real view; its hybrid policy keeps the custom kernel under - # its shape guard (one mode). + # into the response column (solprop SEP 25). A device build's hybrid policy keeps + # the custom kernel under its shape guard (one mode). build = solprop_native.build_info() - if build.device_backend == "none": - expected_provider = "folded" - elif build.gemm_provider == "KOKKOSBLAS_TPL": - expected_provider = ( - str(metadata["synthesis_provider"]).removesuffix("-interleaved-real") - + "-interleaved-real" - ) - assert str(metadata["synthesis_provider"]).endswith("-interleaved-real") - else: - expected_provider = "custom" + expected_provider = "folded" if build.device_backend == "none" else "custom" assert metadata["synthesis_provider"] == expected_provider assert metadata["projection_provider"] == expected_provider assert "every whole transform on the host" in cast( @@ -848,12 +843,9 @@ def test_packed_real_blas_radial_envelope_matches_the_reference_when_eligible() try: state = load_native(backend, problem.initial.field0, clear_fsal=False) info = build_info() - # The native rule: a HYBRID or KOKKOSBLAS_TPL build on a host execution - # space, at this shape; the dev build is HYBRID without TPL BLAS and packs too. - eligible = ( - info.gemm_provider in {"HYBRID", "KOKKOSBLAS_TPL"} - and info.device_backend == "none" - ) + # The native rule: a host execution space, at this shape; the dev build + # without TPL BLAS packs too. + eligible = info.device_backend == "none" provider = str(backend.metadata()["synthesis_provider"]) ledger = backend.memory_ledger() if eligible: @@ -942,12 +934,12 @@ def test_the_plasma_stash_is_off_under_modified_noise( ) problem = simulation.translate() assert problem.responses.plasma_channels - payload = _native_problem( # pyright: ignore[reportPrivateUsage] + payload = _lower( problem, simulation._run_request(), # pyright: ignore[reportPrivateUsage] - terms(problem), + _LowerPlan(terms=terms(problem)), ) - assert payload["stash_plasma"] is stashed + assert payload.plasma.stash is stashed def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: @@ -956,7 +948,7 @@ def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: The last stage's input is then not the committed field, so the plasma statistics are scanned afresh (the stash needs a first-same-as-last step). """ - payloads = {} + payloads: dict[str, _core.Problem] = {} for scheme in ("nonlinear", "split_step"): simulation = Simulation.from_capillary( 125e-6, @@ -982,11 +974,11 @@ def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: simulation.settings, absorber_scheme=scheme ) problem = simulation.translate() - payloads[scheme] = _native_problem( # pyright: ignore[reportPrivateUsage] + payloads[scheme] = _lower( problem, simulation._run_request(), # pyright: ignore[reportPrivateUsage] - terms(problem), + _LowerPlan(terms=terms(problem)), ) - assert payloads["nonlinear"]["stash_plasma"] is True - assert payloads["split_step"]["split_step_absorber"] is True - assert payloads["split_step"]["stash_plasma"] is False + assert payloads["nonlinear"].plasma.stash is True + assert payloads["split_step"].absorber.split_step is True + assert payloads["split_step"].plasma.stash is False diff --git a/native/tests/test_band_plasma.py b/native/tests/test_band_plasma.py index d9da0efcc..bee833d1a 100644 --- a/native/tests/test_band_plasma.py +++ b/native/tests/test_band_plasma.py @@ -62,7 +62,8 @@ from plasma_bounds import assert_regression_guard, plasma_stage_bound, rate_roundoff from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) INFO = solprop_native.build_info() @@ -756,24 +757,25 @@ def test_native_taps_are_validated(damage: str) -> None: grid, problem, run_request = modal("mode_averaged", "band") backend = NativeBackend(problem, run_request, grid=grid) try: - payload = _native_problem( + payload = _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - fixed payload + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] - fixed payload ) finally: backend.close() - taps = np.array(payload["plasma_taps"], dtype=float, copy=True) + plasma = payload.plasma + taps = plasma.taps if damage == "too_many": - payload["plasma_taps"] = np.ones(10) + plasma.taps = np.ones(10) elif damage == "nonfinite": taps[1] = np.nan - payload["plasma_taps"] = taps + plasma.taps = taps elif damage == "empty": - payload["plasma_taps"] = np.empty(0) + plasma.taps = np.empty(0) else: # The trapezoid is the one identity tap and nothing else. - payload["plasma_band"] = False + plasma.band = False with pytest.raises(ValueError, match=r"taps|trapezoid|band"): _core.State(payload) diff --git a/native/tests/test_coupled_flow.py b/native/tests/test_coupled_flow.py index a917518e0..73209941b 100644 --- a/native/tests/test_coupled_flow.py +++ b/native/tests/test_coupled_flow.py @@ -23,7 +23,7 @@ import math import warnings from collections.abc import Iterator -from typing import cast +from typing import Any, cast import numpy as np import numpy.typing as npt @@ -47,10 +47,10 @@ from solprop.translate.problem import PropagationProblem from solprop.translate.run_request import RunRequest -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _flow_payload, # pyright: ignore[reportPrivateUsage] _host_flow, # pyright: ignore[reportPrivateUsage] + _lower_flow, # pyright: ignore[reportPrivateUsage] ) from solprop_native.group import refusal as group_refusal @@ -58,6 +58,13 @@ type RealArray = npt.NDArray[np.float64] +def _flow_rows(flow: Any) -> _core.Problem.Flow: + """The native problem's coupled flow group as the adapter fills it.""" + native = _core.Problem() + _lower_flow(native, flow) + return native.flow + + def _norms(stack: ComplexArray | RealArray, *, spectral: bool = True) -> RealArray: """Return the 2-norm (or Frobenius norm) of each matrix of a stack.""" if spectral: @@ -460,9 +467,9 @@ def test_the_residual_s_data_are_zero_exactly_outside_the_requested_band() -> No scored = problem.grids.inband_mask[flow.active] assert np.any(~scored) assert np.any(scored) - payload = _flow_payload(flow) - frozen = cast("ComplexArray", payload["flow_frozen"]) - terms = cast("ComplexArray", payload["flow_residual_terms"]) + payload = _flow_rows(flow) + frozen = payload.frozen.reshape(flow.frozen.shape) + terms = payload.residual_terms.reshape(flow.residual_terms.shape) assert np.all(frozen[:, ~scored] == 0.0) assert np.all(terms[:, ~scored] == 0.0) assert np.array_equal(frozen[:, scored], flow.frozen[:, scored]) @@ -475,8 +482,8 @@ def test_the_residual_s_data_are_zero_exactly_outside_the_requested_band() -> No ) full = _host_flow(everywhere) assert full is not None - unmasked = _flow_payload(full) - assert np.array_equal(cast("ComplexArray", unmasked["flow_frozen"]), flow.frozen) + unmasked = _flow_rows(full) + assert np.array_equal(unmasked.frozen.reshape(flow.frozen.shape), flow.frozen) assert np.array_equal( - cast("ComplexArray", unmasked["flow_residual_terms"]), flow.residual_terms + unmasked.residual_terms.reshape(flow.residual_terms.shape), flow.residual_terms ) diff --git a/native/tests/test_dispersive.py b/native/tests/test_dispersive.py index ead61de74..c0166f76e 100644 --- a/native/tests/test_dispersive.py +++ b/native/tests/test_dispersive.py @@ -4,7 +4,7 @@ """The adapter of solprop SEP 6's dispersive response. -The native response executes the layout the adapter packs (``_dispersive_payload``): +The native response executes the layout the adapter packs (``_lower_dispersive``): these tests hold the layout to the leaf (the general modal route's regions among it), the field family to the sector and one evaluation to solprop's reference. The full parity ladders are solprop's @@ -41,12 +41,21 @@ from solprop.translate.responses import dispersive_terms from solprop.translate.run_request import RunRequest -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _dispersive_payload, # pyright: ignore[reportPrivateUsage] - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _lower_dispersive, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) + +def _dispersive_payload(problem: PropagationProblem) -> _core.Problem.Dispersive: + """The native problem's dispersive group as the adapter fills it.""" + native = _core.Problem() + _lower_dispersive(native, problem) + return native.dispersive + + EPS = float(np.finfo(float).eps) LENGTH = 2e-3 POLICIES = {"symmetric": "symmetric-kerr", "polynomial": "gas-polynomial"} @@ -120,25 +129,28 @@ def test_a_symmetric_mixture_is_one_product_under_its_filter() -> None: leaf = problem.responses.dispersive_kerr assert leaf is not None assert leaf.n_channels == 2 - payload = cast(dict[str, Any], _dispersive_payload(problem)) + payload = _dispersive_payload(problem) bins = leaf.positive.size - assert payload["dispersive_symmetric"] is True - assert payload["dispersive_form"] == 1 - assert payload["dispersive_inputs"].shape == (1, bins) + assert payload.symmetric is True + assert payload.form == 1 + assert payload.inputs.shape == (bins,) # Copies, indices, masks and power-of-two scalings of a mask are exact; the # quartics are three products of the filters', within gamma(3) each way. - np.testing.assert_array_equal(payload["dispersive_inputs"][0], leaf.filters[0]) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [3]) - np.testing.assert_array_equal(payload["dispersive_group_offsets"], [0, 2]) - np.testing.assert_array_equal(payload["dispersive_group_members"], [0, 1]) + np.testing.assert_array_equal(payload.inputs, leaf.filters[0]) + np.testing.assert_array_equal(payload.term_rows, [0]) + np.testing.assert_array_equal(payload.term_channels, [3]) + np.testing.assert_array_equal(payload.group_offsets, [0, 2]) + np.testing.assert_array_equal(payload.group_members, [0, 1]) squares = leaf.filters * leaf.filters np.testing.assert_allclose( - payload["dispersive_quartics"], squares * squares, rtol=2 * gamma(3), atol=0.0 + payload.quartics.reshape(squares.shape), + squares * squares, + rtol=2 * gamma(3), + atol=0.0, ) positive = leaf.positive.astype(float) - np.testing.assert_array_equal(payload["dispersive_output_scale"], positive / 4.0) - np.testing.assert_array_equal(payload["dispersive_input_scale"], np.ones(bins)) + np.testing.assert_array_equal(payload.output_scale, positive / 4.0) + np.testing.assert_array_equal(payload.input_scale, np.ones(bins)) def test_the_polynomial_groups_species_of_one_leg_bound() -> None: @@ -154,34 +166,34 @@ def test_the_polynomial_groups_species_of_one_leg_bound() -> None: assert leaf is not None assert leaf.groups() == ((0, 1),) assert leaf.outputs is not None - payload = cast(dict[str, Any], _dispersive_payload(problem)) + payload = _dispersive_payload(problem) positive = leaf.positive.astype(float) # The field grid's conserving sector: the analytic family, Pi_+ / 8 on the output. - assert payload["dispersive_form"] == 1 + assert payload.form == 1 # The masks, indices and power-of-two scalings are exact; a product of filters # or of a multiplier and the mask is within gamma(2) each way. - np.testing.assert_array_equal(payload["dispersive_input_scale"], positive) + np.testing.assert_array_equal(payload.input_scale, positive) np.testing.assert_allclose( - payload["dispersive_inputs"], + payload.inputs.reshape(2, -1), np.stack([leaf.filters[0], leaf.filters[0] * leaf.filters[0]]) * leaf.positive, rtol=2 * gamma(2), atol=0.0, ) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0, 1, 0, 1, 2]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [0, 0, 1, 1, 2]) + np.testing.assert_array_equal(payload.term_rows, [0, 1, 0, 1, 2]) + np.testing.assert_array_equal(payload.term_channels, [0, 0, 1, 1, 2]) np.testing.assert_allclose( - payload["dispersive_term_multipliers"][2], + payload.term_multipliers.reshape(-1, positive.size)[2], leaf.outputs[1, 0] * positive / 8.0, rtol=2 * gamma(2), atol=0.0, ) np.testing.assert_array_equal( - payload["dispersive_term_multipliers"][4], positive / 8.0 + payload.term_multipliers.reshape(-1, positive.size)[4], positive / 8.0 ) np.testing.assert_array_equal( - payload["dispersive_pair_constants"], np.asarray(leaf.products) + payload.pair_constants.reshape(-1, 2), np.asarray(leaf.products) ) - assert payload["dispersive_quartics"].size == 0 + assert payload.quartics.size == 0 @pytest.mark.parametrize( @@ -200,17 +212,19 @@ def test_the_sector_chooses_the_field_family( problem, run_request = capillary("gas-polynomial", envelope=envelope, thg=thg) backend = NativeBackend(problem, run_request) try: - payload = _native_problem(problem, run_request, backend._terms) # pyright: ignore[reportPrivateUsage] + payload = _lower(problem, run_request, _LowerPlan(terms=backend._terms)) # pyright: ignore[reportPrivateUsage] finally: backend.close() - assert payload["family"] == family - assert payload["dispersive_form"] == form - assert payload["nonlinear"] is False + assert payload.dims.family == getattr(_core.FieldFamily, family) + assert payload.dispersive.form == form + assert payload.kerr.nonlinear is False def test_a_problem_without_the_leaf_packs_nothing() -> None: problem, _ = capillary("carrier-kerr", envelope=False, thg=True) - assert _dispersive_payload(problem) == {"dispersive": False} + payload = _dispersive_payload(problem) + assert payload.enabled is False + assert payload.inputs.size == 0 def general_leaf(evaluator: str) -> tuple[Any, FieldGrid]: @@ -247,23 +261,19 @@ def test_the_general_route_lays_out_its_regions(evaluator: str) -> None: ), grid_kind="field", ) - payload = cast( - dict[str, Any], _dispersive_payload(cast(PropagationProblem, problem)) - ) - np.testing.assert_array_equal(payload["dispersive_node_region"], region_index) - assert payload["dispersive_form"] == 1 + payload = _dispersive_payload(cast(PropagationProblem, problem)) + np.testing.assert_array_equal(payload.node_region, region_index) + assert payload.form == 1 if evaluator == "symmetric": assert leaf.groups() == ((0,), (1, 2)) - np.testing.assert_array_equal(payload["dispersive_group_keys"], [3, 5]) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0, 1]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [0, 4]) + np.testing.assert_array_equal(payload.group_keys, [3, 5]) + np.testing.assert_array_equal(payload.term_rows, [0, 1]) + np.testing.assert_array_equal(payload.term_channels, [0, 4]) return assert leaf.groups() == ((0,), (1,), (2,)) - np.testing.assert_array_equal(payload["dispersive_group_keys"], [3, 5, 5]) + np.testing.assert_array_equal(payload.group_keys, [3, 5, 5]) # Two cubes, by region in the order the groups name them; then each group's pair. - np.testing.assert_array_equal( - payload["dispersive_term_rows"], [0, 2, 3, 1, 4, 5, 1, 6, 7] - ) + np.testing.assert_array_equal(payload.term_rows, [0, 2, 3, 1, 4, 5, 1, 6, 7]) def test_a_trial_without_the_densities_is_refused() -> None: diff --git a/native/tests/test_envelope_plasma.py b/native/tests/test_envelope_plasma.py index 80a9fb5c4..e12dc990a 100644 --- a/native/tests/test_envelope_plasma.py +++ b/native/tests/test_envelope_plasma.py @@ -15,7 +15,6 @@ from typing import Any, cast import numpy as np -import numpy.typing as npt import pytest from legacy_geometry import LEGACY_GRID from session_runs import ( @@ -49,7 +48,8 @@ from solprop_native import NativeBackend, _core, build_info from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - validate the native boundary + _lower, # pyright: ignore[reportPrivateUsage] - validate the native boundary + _LowerPlan, # pyright: ignore[reportPrivateUsage] - validate the native boundary ) # The FFT collar extends beyond the neutral-gas index fit. These are current @@ -196,10 +196,7 @@ def test_envelope_current_streams_use_large_radial_projection() -> None: native_trial(actual, 0.0, 0.0) observed = np.asarray(state.stage_to_host(0)).reshape(problem.shape) info = build_info() - if info.device_backend == "none" and info.gemm_provider in { - "HYBRID", - "KOKKOSBLAS_TPL", - }: + if info.device_backend == "none": assert str(actual.metadata()["projection_provider"]).endswith( "-packed-real" ) @@ -378,38 +375,46 @@ def test_envelope_currents_steps_statistics_and_repeatability( ) def test_envelope_native_payload_rejects_inconsistent_inputs(corruption: str) -> None: problem, run_request, _ = envelope_case("modal", 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options("modal").chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options("modal").chunk), ) - if corruption in ("field", "thg", "stash"): - key, value = { - "field": ("family", "field_real"), - "thg": ("harmonic_factor", 1.0), - "stash": ("stash_plasma", True), - }[corruption] - payload[key] = value - elif corruption in ("shape", "nan", "weight"): - key = ( - "envelope_rule_nodes" if corruption != "weight" else "envelope_rule_weights" - ) - array = cast(npt.NDArray[np.float64], payload[key]).copy() - if corruption == "shape": - payload[key] = array.ravel() + plasma = payload.plasma + shape = (plasma.n_channels, 4, 512) + + def _spoil() -> None: + if corruption == "field": + payload.dims.family = _core.FieldFamily.field_real + elif corruption == "thg": + payload.kerr.harmonic_factor = 1.0 + elif corruption == "stash": + plasma.stash = True + elif corruption in ("shape", "nan", "weight"): + weight = corruption == "weight" + rule = ( + plasma.envelope_rule_weights if weight else plasma.envelope_rule_nodes + ) + if corruption == "shape": + corrupted = rule + else: + corrupted = rule.reshape(shape) + corrupted.flat[0] = np.nan if corruption == "nan" else 0.0 + if weight: + plasma.envelope_rule_weights = corrupted + else: + plasma.envelope_rule_nodes = corrupted else: - array.flat[0] = np.nan if corruption == "nan" else 0.0 - payload[key] = array - else: - array = cast(npt.NDArray[np.float64], payload["envelope_drude_factor"]).copy() - mask = problem.grids.active_mask & (problem.grids.omega > 0) - index = int(np.flatnonzero(mask if corruption == "sign" else ~mask)[0]) - array[index] = 1.0 - payload["envelope_drude_factor"] = array - with pytest.raises(ValueError, match="envelope"): + array = plasma.envelope_drude_factor + mask = problem.grids.active_mask & (problem.grids.omega > 0) + index = int(np.flatnonzero(mask if corruption == "sign" else ~mask)[0]) + array[index] = 1.0 + plasma.envelope_drude_factor = array _core.State(payload) + with pytest.raises(ValueError, match="envelope"): + _spoil() + @pytest.mark.parametrize( ("geometry", "n_pol"), [("modal", 1), ("radial", 2), ("free", 2)] @@ -427,24 +432,23 @@ def test_envelope_angular_failure_is_a_solprop_convergence_error( # No statistics: the failure is the first step's, inside the native march, as the # driver's was (the launch plane's populations would meet the bad rule first). run_request = run_request.for_run(stats=StatsSpec()) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # Sorted interior nodes and normalized positive weights still satisfy native # shape/domain checks. Alternating incompatible rules must fail refinement. for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) ) diff --git a/native/tests/test_envelope_statistics_refusal.py b/native/tests/test_envelope_statistics_refusal.py index 0ee929ddc..9116b28a5 100644 --- a/native/tests/test_envelope_statistics_refusal.py +++ b/native/tests/test_envelope_statistics_refusal.py @@ -11,10 +11,7 @@ from __future__ import annotations -from typing import cast - import numpy as np -import numpy.typing as npt import pytest from session_runs import drain, run from solprop.backends.events import Outcome @@ -24,9 +21,10 @@ from test_envelope_plasma import envelope_case, envelope_options, terms import solprop_native.backend as adapter -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) SPEC = StatsSpec(names=("electron_density", "peak_ionisation_rate")) @@ -40,23 +38,22 @@ def test_the_statistics_refuse_and_a_quiet_field_recovers( monkeypatch: pytest.MonkeyPatch, geometry: str ) -> None: problem, run_request, grid = envelope_case(geometry, 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # As the source's test: alternating incompatible rules must fail refinement. for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) # The launch plane's statistics meet the bad rule before any step. run_request = run_request.for_run(stats=SPEC) owner = NativeBackend( diff --git a/native/tests/test_execution.py b/native/tests/test_execution.py index 474661da8..705ea3243 100644 --- a/native/tests/test_execution.py +++ b/native/tests/test_execution.py @@ -24,7 +24,8 @@ from solprop_native._wait import SPIN_MILLISECONDS from solprop_native.backend import ( NativeBackend, - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) # The constants of core.hpp before SEP 46 moved them here (solprop PR #576), Linux and @@ -175,27 +176,59 @@ def test_records_round_trip_and_name_their_differences() -> None: assert table.differences(table) == {} -def _payload(case: PhaseACase) -> dict[str, object]: +_MISSING = object() +"""No record: the state is built without one.""" + + +def _payload(case: PhaseACase) -> _core.Problem: problem, run_request, grid = case backend = NativeBackend(problem, run_request, grid=grid) try: - return _native_problem(problem, run_request, backend._terms) # pyright: ignore[reportPrivateUsage] + return _lower(problem, run_request, _LowerPlan(terms=backend._terms)) # pyright: ignore[reportPrivateUsage] finally: backend.close() +def _with_record(payload: _core.Problem, record: object = _MISSING) -> _core.Problem: + """A copy of `payload` with its CPU execution record replaced, or without one.""" + changed = _core.Problem() + for group in _GROUPS: + setattr(changed, group, getattr(payload, group)) + if record is not _MISSING: + changed.cpu_execution = record # pyright: ignore[reportAttributeAccessIssue] + return changed + + +_GROUPS = ( + "dims", + "grid", + "spatial", + "flux", + "stepper", + "linear", + "absorber", + "transform", + "kerr", + "plasma", + "raman", + "general", + "flow", + "dispersive", +) + + def test_a_state_reads_back_the_record_it_was_built_with( phase_a_case: PhaseACase, ) -> None: payload = _payload(phase_a_case) - assert payload["cpu_execution"] == native.BUILTIN_EXECUTION.native() + assert payload.cpu_execution == native.BUILTIN_EXECUTION.native() serial = dataclasses.replace( native.BUILTIN_EXECUTION, **{f"{name}_threshold": 1 << 40 for name in ("stage", "copy", "adams")}, **{f"{name}_full": 1 << 40 for name in ("stage", "copy", "adams")}, ) first = _core.State(payload) - second = _core.State(payload | {"cpu_execution": serial.native()}) + second = _core.State(_with_record(payload, serial.native())) try: # Two live workspaces hold their own records (solprop SEP 46 section 7 item 7). assert first.cpu_execution == native.BUILTIN_EXECUTION.native() @@ -223,13 +256,16 @@ def test_a_state_refuses_a_missing_or_malformed_record( phase_a_case: PhaseACase, change: dict[str, object] | str, message: str ) -> None: payload = _payload(phase_a_case) - if change == "missing": - del payload["cpu_execution"] - else: - assert isinstance(change, dict) - payload |= change + + def _spoil() -> None: + if change == "missing": + _core.State(_with_record(payload)) + else: + assert isinstance(change, dict) + _core.State(_with_record(payload, change["cpu_execution"])) + with pytest.raises(ValueError, match=message): - _core.State(payload) + _spoil() def test_a_state_refuses_a_record_the_native_core_refuses( @@ -238,12 +274,12 @@ def test_a_state_refuses_a_record_the_native_core_refuses( payload = _payload(phase_a_case) record = native.BUILTIN_EXECUTION.native() with pytest.raises(ValueError, match="positive threshold"): - _core.State(payload | {"cpu_execution": record | {"dense_full": 1}}) + _core.State(_with_record(payload, record | {"dense_full": 1})) with pytest.raises(ValueError, match="positive response bounds"): - _core.State(payload | {"cpu_execution": record | {"medium_threads": 0}}) + _core.State(_with_record(payload, record | {"medium_threads": 0})) renamed = {k: v for k, v in record.items() if k != "adams_full"} with pytest.raises(ValueError, match="misses a field: adams_full"): - _core.State(payload | {"cpu_execution": renamed | {"adams_fulll": 1}}) + _core.State(_with_record(payload, renamed | {"adams_fulll": 1})) def test_a_backend_given_a_record_builds_with_it_and_records_it( diff --git a/native/tests/test_free.py b/native/tests/test_free.py index a05a10f79..ea8fd6535 100644 --- a/native/tests/test_free.py +++ b/native/tests/test_free.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import cast +from typing import Any import numpy as np import pytest @@ -26,12 +26,42 @@ from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) +_GROUP = { + "n_x": "spatial", + "n_y": "spatial", + "n_p": "spatial", + "n_omega": "dims", + "n_to": "dims", + "node_chunk": "dims", + "n_pol_state": "dims", + "projection_matrix": "transform", + "projection_coarse_matrix": "transform", +} + + +def _path(name: str) -> tuple[str, str]: + """The native problem's group and field of a flat payload name.""" + if name.startswith("spatial_"): + return "spatial", name.removeprefix("spatial_") + return _GROUP[name], name + + +def _field(payload: _core.Problem, name: str) -> Any: + group, field = _path(name) + return getattr(getattr(payload, group), field) + + +def _set(payload: _core.Problem, name: str, value: object) -> None: + group, field = _path(name) + setattr(getattr(payload, group), field, value) + @pytest.fixture -def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: +def free_payload(phase_a_case: PhaseACase) -> _core.Problem: _, base_request, grid = phase_a_case space = FreeGrid(1e-3, 4, 1.5e-3, 6, collar_fraction=0) problem, run_request = assemble( @@ -50,11 +80,13 @@ def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: problem, run_request, grid=grid, options=ExecutionOptions(node_chunk=5) ) try: - return _native_problem( + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - node_chunk=backend.node_chunk, + _LowerPlan( + terms=backend._terms, # pyright: ignore[reportPrivateUsage] + node_chunk=backend.node_chunk, + ), ) finally: backend.close() @@ -80,19 +112,23 @@ def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: ) @pytest.mark.parametrize("spoiling", ["length", "nonfinite", "dtype"]) def test_free_native_vectors_reject_invalid_data( - free_payload: dict[str, object], name: str, spoiling: str + free_payload: _core.Problem, name: str, spoiling: str ) -> None: - values = np.array(cast(np.ndarray, free_payload[name]), copy=True) + values = np.array(_field(free_payload, name), copy=True) if spoiling == "length": values = values[:-1] elif spoiling == "nonfinite": values[0] = np.nan else: values = values.astype(np.complex64 if np.iscomplexobj(values) else np.float32) - free_payload[name] = values - with pytest.raises((TypeError, ValueError)): + + def _spoil() -> None: + _set(free_payload, name, values) _core.State(free_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + @pytest.mark.parametrize( ("name", "value"), @@ -106,33 +142,37 @@ def test_free_native_vectors_reject_invalid_data( ], ) def test_free_native_axes_and_clamp_are_checked( - free_payload: dict[str, object], name: str, value: float + free_payload: _core.Problem, name: str, value: float ) -> None: - free_payload[name] = value - with pytest.raises((TypeError, ValueError)): + def _spoil() -> None: + _set(free_payload, name, value) _core.State(free_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + def test_the_far_field_payload_belongs_to_free_space( - free_payload: dict[str, object], planar_payload: dict[str, object] + free_payload: _core.Problem, planar_payload: _core.Problem ) -> None: """solprop SEP 23: the axes and the slot count are free space's alone.""" - assert free_payload["spatial_far_field_slots"] == 0 - assert cast(np.ndarray, free_payload["spatial_k_x"]).shape == (4,) - assert cast(np.ndarray, free_payload["spatial_k_y"]).shape == (6,) + assert free_payload.spatial.far_field_slots == 0 + assert free_payload.spatial.k_x.shape == (4,) + assert free_payload.spatial.k_y.shape == (6,) + planar_payload.spatial.far_field_slots = 3 with pytest.raises(ValueError, match="belong to free space"): - _core.State(planar_payload | {"spatial_far_field_slots": 3}) + _core.State(planar_payload) def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: state = _core.State(free_payload) try: ledger = state.memory_ledger() omega, time = ( - int(cast(int, free_payload["n_omega"])), - int(cast(int, free_payload["n_to"])), + free_payload.dims.n_omega, + free_payload.dims.n_to, ) assert ledger["transverse_scratch"] == 24 * omega * 16 assert ledger["modal_time"] == 5 * time * 16 @@ -141,7 +181,7 @@ def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( assert ledger["fixed_phi_terms"] == ledger["fixed_prefactor_terms"] == 0 assert ledger["transform_matrices"] == 0 expected = sum( - cast(np.ndarray, free_payload[name]).nbytes + _field(free_payload, name).nbytes for name in ( "spatial_k_squared", "spatial_k_squared_low", @@ -163,12 +203,12 @@ def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( @pytest.mark.parametrize("slots", [3, 7, 8]) def test_adams_dense_storage_is_optional_and_reallocation_is_idempotent( - free_payload: dict[str, object], slots: int + free_payload: _core.Problem, slots: int ) -> None: state = _core.State(free_payload) try: field_bytes = state.memory_ledger()["state"] - omega = int(cast(int, free_payload["n_omega"])) + omega = free_payload.dims.n_omega initial = np.zeros(field_bytes // 16, dtype=np.complex128) state.set_initial(initial, np.empty(0, dtype=np.complex128)) for dense in (False, True, True, False): @@ -192,7 +232,7 @@ def test_adams_dense_storage_is_optional_and_reallocation_is_idempotent( def test_adams_rows_cross_to_the_host_and_back_exactly( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: # solprop SEP 29 G2: the resident history's rows are exported for a resume and # installed again; a copy, so exact, and refused by name, extent, dtype and order @@ -200,7 +240,7 @@ def test_adams_rows_cross_to_the_host_and_back_exactly( state = _core.State(free_payload) try: columns = state.memory_ledger()["state"] // 16 - omega = int(cast(int, free_payload["n_omega"])) + omega = free_payload.dims.n_omega state.set_initial( np.zeros(columns, dtype=np.complex128), np.empty(0, dtype=np.complex128) ) @@ -387,18 +427,20 @@ def test_the_hybrid_planar_kind_matches_the_reference(phase_a_case: PhaseACase) @pytest.fixture -def planar_payload(phase_a_case: PhaseACase) -> dict[str, object]: +def planar_payload(phase_a_case: PhaseACase) -> _core.Problem: _, base_request, grid = phase_a_case space, problem, run_request = _planar_problem(grid, base_request.stepper.controller) backend = NativeBackend( problem, run_request, grid=grid, options=_planar_options(space) ) try: - return _native_problem( + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - node_chunk=backend.node_chunk, + _LowerPlan( + terms=backend._terms, # pyright: ignore[reportPrivateUsage] + node_chunk=backend.node_chunk, + ), ) finally: backend.close() @@ -418,32 +460,34 @@ def planar_payload(phase_a_case: PhaseACase) -> dict[str, object]: ], ) def test_the_planar_native_payload_is_checked( - planar_payload: dict[str, object], name: str, value: int | str + planar_payload: _core.Problem, name: str, value: int | str ) -> None: """The planar axes, whole-row chunks, scalar state and wall terms are validated.""" - kappa = np.array(cast(np.ndarray, planar_payload["spatial_kappa_sq"]), copy=True) - if value == "present": - planar_payload[name] = np.array( - cast(np.ndarray, planar_payload["projection_matrix"]), copy=True - ) - elif value == "short": - planar_payload[name] = kappa[:-1] - elif value == "nonfinite": - kappa[0] = np.nan - planar_payload[name] = kappa - else: - planar_payload[name] = value - with pytest.raises((TypeError, ValueError)): + kappa = planar_payload.spatial.kappa_sq + + def _spoil() -> None: + if value == "present": + _set(planar_payload, name, planar_payload.transform.projection_matrix) + elif value == "short": + _set(planar_payload, name, kappa[:-1]) + elif value == "nonfinite": + kappa[0] = np.nan + _set(planar_payload, name, kappa) + else: + _set(planar_payload, name, value) _core.State(planar_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + def test_a_free_payload_with_slab_channels_is_refused( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: """Slab channels turn a plane into the planar kind, whose shapes it lacks.""" - free_payload["n_p"] = 1 - free_payload["spatial_kappa_sq"] = np.zeros( - int(cast(int, free_payload["n_omega"])), dtype=np.complex128 + free_payload.spatial.n_p = 1 + free_payload.spatial.kappa_sq = np.zeros( + free_payload.dims.n_omega, dtype=np.complex128 ) with pytest.raises(ValueError, match="hybrid planar"): _core.State(free_payload) diff --git a/native/tests/test_mixtures.py b/native/tests/test_mixtures.py index 248e54583..aedda1934 100644 --- a/native/tests/test_mixtures.py +++ b/native/tests/test_mixtures.py @@ -22,7 +22,8 @@ from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) @@ -96,14 +97,15 @@ def test_native_channel_payload_refusals( problem, run_request, grid = mixture_case backend = NativeBackend(problem, run_request, grid=grid) try: - payload = _native_problem( + payload = _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - fixed payload + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] - fixed payload ) - parameters = np.array(payload["channel_parameters"], dtype=float, copy=True) + plasma = payload.plasma + parameters = plasma.channel_parameters.reshape(plasma.n_channels, 9) if damage == "zero_channels": - payload["n_channels"] = 0 + plasma.n_channels = 0 elif damage == "parameter_extent": parameters = parameters[:, :-1].copy() elif damage == "nonfinite": @@ -113,8 +115,8 @@ def test_native_channel_payload_refusals( elif damage == "interval": parameters[1, 5] = -1 else: - payload["channel_coefficients"] = np.empty(0) - payload["channel_parameters"] = parameters + plasma.channel_coefficients = np.empty(0) + plasma.channel_parameters = parameters with pytest.raises(ValueError, match=r"channel|Channel|plasma"): _core.State(payload) finally: diff --git a/native/tests/test_radial.py b/native/tests/test_radial.py index 3b6378b02..d4f8314ec 100644 --- a/native/tests/test_radial.py +++ b/native/tests/test_radial.py @@ -36,7 +36,7 @@ def test_radial_short_envelope_uses_device_blas( n_r: int, n_pol: int, chunk: int, eligible: bool ) -> None: info = build_info() - if info.device_backend == "none" or info.gemm_provider != "HYBRID": + if info.device_backend == "none": pytest.skip("default device HYBRID dispatch qualification") if not info.tpl_blas_enabled: pytest.skip("device vendor BLAS is disabled") @@ -123,7 +123,7 @@ def test_radial_partial_chunks_preserve_the_reference_source( # An analytic field synthesises the retained spectrum, on a host and on a # device; the others keep their time-domain products. info = build_info() - retained = family == "analytic" and info.gemm_provider != "CUSTOM" + retained = family == "analytic" assert info.device_backend in {"none", "cuda", "hip"} assert ( str(backend.metadata()["synthesis_provider"]).endswith( diff --git a/native/tests/test_session_failure.py b/native/tests/test_session_failure.py index 3197d57f9..a3fd0e517 100644 --- a/native/tests/test_session_failure.py +++ b/native/tests/test_session_failure.py @@ -13,10 +13,7 @@ from __future__ import annotations -from typing import cast - import numpy as np -import numpy.typing as npt import pytest from solprop.backends.events import Terminal from solprop.backends.session import Start @@ -24,9 +21,10 @@ from test_envelope_plasma import envelope_case, envelope_options, terms import solprop_native.backend as adapter -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) @@ -35,23 +33,22 @@ def test_an_unresolved_launch_statistic_ends_the_run_as_a_convergence_error( monkeypatch: pytest.MonkeyPatch, geometry: str ) -> None: problem, run_request, grid = envelope_case(geometry, 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # Alternating incompatible rules fail refinement (`test_envelope_plasma.py`). for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) ) diff --git a/pyproject.toml b/pyproject.toml index b1b82c9e7..d0cacbc6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,11 +81,15 @@ dev = [ "drawsvg>=2.4", "ruff>=0.16.6", "psutil>=7.1.0", - # The native gate (native/scripts/check.sh). The C++ formatter is pinned so the check - # does not depend on the machine's LLVM (Ubuntu's 18 and Homebrew's 23 format - # native/src/general.cpp differently); the build tools let it build the editable - # package and the distribution wheel without isolation. + # The native gate (native/scripts/check.sh). The C++ formatter and clang-tidy are + # pinned so the check does not depend on the machine's LLVM (Ubuntu's 18 and + # Homebrew's 23 format native/src/general.cpp differently, and each LLVM release + # adds checks); clang-tidy's PyPI wheels start at 23.1.3, the nearest to the + # formatter's 23.1.0, and exist for Linux and macOS (the gate is bash) but not + # Windows on Arm, where a sync would build LLVM. The build tools let it build the + # editable package and the distribution wheel without isolation. "clang-format==23.1.0", + "clang-tidy==23.1.4; sys_platform == 'linux' or sys_platform == 'darwin'", "nanobind>=2.9", "scikit-build-core>=0.11", "setuptools-scm>=9", @@ -459,6 +463,19 @@ help = "The definition-of-done gate, in order." cmd = "bash native/scripts/check.sh" help = "The native package's gate (C++ and its Python adapter)." +# The pinned clang-format over every tracked C++ file under native/ (the lint stage of +# check-native checks the same list). +[tool.poe.tasks.fmt-native] +shell = "git ls-files -z -- 'native/*.cpp' 'native/*.hpp' 'native/*.h' | xargs -0 clang-format -i" +interpreter = "bash" +help = "Format every tracked native C++ file with the pinned clang-format." + +# The pinned clang-tidy over the given native C++ files, or those that differ from main, +# with the cpu-tidy preset's compile commands (native/scripts/tidy.sh). +[tool.poe.tasks.tidy-native] +cmd = "bash native/scripts/tidy.sh" +help = "clang-tidy on the given native C++ files, or on those changed from main." + [tool.poe.tasks.fetch-wheel] cmd = "python native/scripts/fetch_wheel.py" help = "Install the native wheel CI built for this checkout; needs `gh`, and no compiler." diff --git a/tests/backends/test_native_far_field_measure.py b/tests/backends/test_native_far_field_measure.py index e6e25f4d5..1c48af486 100644 --- a/tests/backends/test_native_far_field_measure.py +++ b/tests/backends/test_native_far_field_measure.py @@ -5,7 +5,7 @@ """A plane measure the far-field pass can't read is refused on ``native`` (SEP 43). ``numba_cpu``'s pass read the operator's root as the plane's, and any other measure was -refused. Native has that guard at the payload (``_flux_payload``), and no test held it. +refused. Native has that guard at the payload (``_lower_flux``), and no test held it. """ import copy diff --git a/tests/backends/test_native_phi_moments.py b/tests/backends/test_native_phi_moments.py index 24c3a7408..67c5703c3 100644 --- a/tests/backends/test_native_phi_moments.py +++ b/tests/backends/test_native_phi_moments.py @@ -91,16 +91,19 @@ def read_moments(module, grid, values): try: # The payload and the step's resident rows are the backend's own; the state # they open is this test's, so nothing is loaded into the backend's. - payload = backend_module._native_problem( - problem, run_request, backend._terms, None, None + payload = backend_module._lower( + problem, run_request, backend_module._LowerPlan(terms=backend._terms) ) arrays, material = backend._resident_arrays(0.0, 1e-3) finally: backend.close() - terms = np.zeros_like(payload["phi_terms"]) + terms = np.zeros_like(payload.linear.phi_terms).reshape( + payload.linear.n_phi_terms, -1, payload.dims.n_omega + ) assert terms.shape[1] == 1 terms[0, 0, bins] = values - state = backend_module._core.State(payload | {"phi_terms": terms}) + payload.linear.phi_terms = terms + state = backend_module._core.State(payload) try: (phi_delta, *rest) = arrays step = np.zeros_like(phi_delta) diff --git a/tests/backends/test_node_chunk.py b/tests/backends/test_node_chunk.py index b5ac30c1f..71d56dc17 100644 --- a/tests/backends/test_node_chunk.py +++ b/tests/backends/test_node_chunk.py @@ -552,14 +552,16 @@ def test_a_general_guide_takes_the_plan_s_budget(native_backend): assert general_planned_chunk(problem, forced) == 2 whole = _general_maps(problem, automatic) assert whole is not None and whole.node_chunk > 1 - payload = native_backend._native_problem( - problem, run_request, terms(problem), general_chunk=2 + payload = native_backend._lower( + problem, + run_request, + native_backend._LowerPlan(terms=terms(problem), general_chunk=2), ) - assert payload["general_node_chunk"] == 2 + assert payload.general.node_chunk == 2 assert ( - native_backend._native_problem(problem, run_request, terms(problem))[ - "general_node_chunk" - ] + native_backend._lower( + problem, run_request, native_backend._LowerPlan(terms=terms(problem)) + ).general.node_chunk == whole.node_chunk ) diff --git a/tests/backends/test_spatial_execution.py b/tests/backends/test_spatial_execution.py index 938757879..1874c7bb3 100644 --- a/tests/backends/test_spatial_execution.py +++ b/tests/backends/test_spatial_execution.py @@ -76,11 +76,10 @@ def test_radial_real_blas_preserves_transform_and_memory_contract( try: metadata, ledger = backend.metadata(), backend.memory_ledger() if backend.name == "native": - # Portable/custom and device builds keep their existing provider. + # Device builds keep their existing provider. packed = str(metadata["synthesis_provider"]).endswith("-packed-real") eligible = ( metadata["device_backend"] == "none" - and metadata["gemm_provider"] in {"HYBRID", "KOKKOSBLAS_TPL"} and n_pol == 1 and chunk == n_r and n_r >= 256 diff --git a/uv.lock b/uv.lock index 28b7d8a1a..aa7f09e75 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.14" [manifest] @@ -228,6 +228,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/e0/d44445801720bb21d9d8fb6a2c16f78065dc1f0d7c9379c5214dd07a2db8/clang_format-23.1.0-py2.py3-none-win_arm64.whl", hash = "sha256:6b565baa8d478eaac44a8a68802cce1e6e998e9c80680d222862761d1afe62e1", size = 1410985, upload-time = "2026-08-26T19:54:09.659Z" }, ] +[[package]] +name = "clang-tidy" +version = "23.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/f1/ce2407dc5ca76d5ada9449fa4203ac563f4f563224f29fa06bcd88a3959a/clang_tidy-23.1.4.tar.gz", hash = "sha256:71ec19e17c485cdb1d80f9c631f525b0b0e5f8a0429f164df75b3869ae61ab44", size = 12404, upload-time = "2026-10-26T08:37:15.954Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/18/581961a934c9df0861fa91dcf2e6c553e1d4e194de10fd80c1c633b731be/clang_tidy-23.1.4-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:16be65b3396d096826cda8b94e7734913b4e2ffe0053fac6b8ccb2efc8d68cb1", size = 33090266, upload-time = "2026-10-26T08:36:39.686Z" }, + { url = "https://files.pythonhosted.org/packages/0c/59/637ce45a6cfa15ddb926e1feeee8077b9ebeb2956548375f24e16d51a908/clang_tidy-23.1.4-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:08e977575e9f800c1c806f2001a3cdfd01323491aec05e86dd24be9a5c2585f4", size = 32338052, upload-time = "2026-10-26T08:36:43.098Z" }, + { url = "https://files.pythonhosted.org/packages/7d/95/ba9463f94319dddd0c3a9d3f65d432ddf71fe9b203dcd7ea63bcff8999e2/clang_tidy-23.1.4-py2.py3-none-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87596cd77dedfdea23ced1d13e57e869dcaee99745bfe942796196da3c197671", size = 43724944, upload-time = "2026-10-26T08:36:46.323Z" }, + { url = "https://files.pythonhosted.org/packages/80/85/9b483f300124a9245ec6da8975612f1841e26da13b27ad8d17f2006e13bd/clang_tidy-23.1.4-py2.py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bb31e4f319ecbe14acb75f23c3050d0f1d4b1d92ea1c66091bc76e4eac2c1ac", size = 45608750, upload-time = "2026-10-26T08:36:49.483Z" }, + { url = "https://files.pythonhosted.org/packages/af/32/4856b0bb395031bb1b8ef6866747ced3428006f05e3b26db0a4eadb57405/clang_tidy-23.1.4-py2.py3-none-manylinux_2_28_i686.whl", hash = "sha256:84b9021905a851948c7f0735baea4d00a7fffd87437bdd106a339e76f6cd7b13", size = 51180158, upload-time = "2026-10-26T08:36:53.432Z" }, + { url = "https://files.pythonhosted.org/packages/3c/c0/28da9c922d84e968e67fb826c4c3a6e3a631ef667474df858c5ca921a420/clang_tidy-23.1.4-py2.py3-none-manylinux_2_31_armv7l.whl", hash = "sha256:8d20ac0cf4dbed1526493e02316796bffc075e05e393519e953261909ba2c8e6", size = 41813066, upload-time = "2026-10-26T08:36:56.54Z" }, + { url = "https://files.pythonhosted.org/packages/32/46/c8487562afaabba832be98bc4eeb39a13b5f5f5a270b4e2547043ada4ea5/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:900276dcb577ed8cb411958efc64feb1ba7334e3c9e70ea5d2e6b5c29aa5335f", size = 42757090, upload-time = "2026-10-26T08:37:00.151Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2b/1103bf0e0ae41f3682a4825ce6c7b6290852c4677b2aa4e632b2c43ac60c/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_i686.whl", hash = "sha256:f428092411271394559c70f7bed218e20409bbd2ee16d0c4cc93f7c6361a595a", size = 53434463, upload-time = "2026-10-26T08:37:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/0b/89/f8a6c51ce1caf68ad4d9b1abf2f8b3b81490f8c9d959c6b3fee1c490d64b/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7c34a8ec573e747412a15a4fea3cd65221b03b2da9a6462f17cbc8bfd47dc7fb", size = 48308787, upload-time = "2026-10-26T08:37:07.276Z" }, +] + [[package]] name = "click" version = "8.5.0" @@ -1581,6 +1598,7 @@ dev = [ { name = "basedpyright" }, { name = "beartype" }, { name = "clang-format" }, + { name = "clang-tidy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "codespell" }, { name = "drawsvg" }, { name = "hypothesis" }, @@ -1631,6 +1649,7 @@ dev = [ { name = "basedpyright", specifier = "==1.39.10" }, { name = "beartype" }, { name = "clang-format", specifier = "==23.1.0" }, + { name = "clang-tidy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'", specifier = "==23.1.4" }, { name = "codespell" }, { name = "drawsvg", specifier = ">=2.4" }, { name = "hypothesis" },