From 2c35944d4d24c26d1beddc10552c7116e36034b8 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:05:11 +0100 Subject: [PATCH 1/6] test_without_petsc4py: hold DistMat's products to a rounding bound, the scatters to zero test_building_blocks_agree_with_petsc asserted that DistMat's mult and multTranspose equal PETSc's exactly. They do on every conda build (all CI legs), but the fleet nightly's pip-built PETSc on aarch64 (spark2, GCC -g -O) differs by one or two ulps, and the four rank counts failed there every night: spark2 dev_pip, 1/2/3/5 ranks: mult = multTranspose = 8.9e-16, 8.9e-16, 4.4e-16, 8.9e-16 GCC contracts a*b+c into a fused multiply-add by default in GNU C mode on aarch64; numpy's sum does not, so the last bit can differ. Neither is wrong. distmat_mpi.py now prints, for each product, k*eps*max_i sum_j |a_ij||x_j| (k the most terms in one sum): two evaluations of a k-term sum each lie within (k/2)*eps*sum|a||x| of the exact one, whatever the order or contraction. The test holds mult and multTranspose to that bound and keeps the vector scatters (forward insert, its truth, reverse add) at exactly zero: they only copy and add exactly representable values. bound, 1/2/3/5 ranks: mult 5.1e-15, 6.3e-15, 6.3e-15, 8.6e-15; multTranspose 2.2e-14, 2.3e-14, 3.5e-14, 2.0e-14 spark2 dev_pip (this helper run with its proteus): differences as above, all under the bound; chewbacca (conda, py3.13): differences 0, 4 passed Co-Authored-By: Claude Opus 5.5 --- test/multilevel/distmat_mpi.py | 25 ++++++++++++++++++++---- test/multilevel/test_without_petsc4py.py | 13 ++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/test/multilevel/distmat_mpi.py b/test/multilevel/distmat_mpi.py index f334be149d..dde388e37e 100644 --- a/test/multilevel/distmat_mpi.py +++ b/test/multilevel/distmat_mpi.py @@ -4,7 +4,14 @@ as a LinearAlgebraTools.DistMat, and compares mult and multTranspose on random vectors, plus a ghosted ParVec_mpi4py against ParVec_petsc4py for the forward insert and reverse add. Needs petsc4py (it is the reference). Prints the -largest differences. +largest differences, and for the two products a bound on what rounding alone +can make them: k*eps*max_i sum_j |a_ij||x_j|, k the most terms in one sum. Two +evaluations of a k-term sum each lie within (k/2)*eps*sum|a||x| of the exact one +whatever their order or whether a multiply and an add are fused, which a PETSc +built by GCC for aarch64 does (GCC contracts a*b+c into an FMA by default in GNU C +mode; spark2's pip PETSc is built with -g -O). On x86-64 without -mfma it cannot. +The vector scatters only copy and add exactly representable values, so they +must agree exactly. Usage: distmat_mpi.py NROWS NCOLS """ @@ -41,8 +48,17 @@ pm.assemblyBegin(); pm.assemblyEnd() dm.assemblyBegin(); dm.assemblyEnd() -x = rng.standard_normal(nCols)[colOff[rank]:colOff[rank + 1]] -y = rng.standard_normal(nRows)[rowOff[rank]:rowOff[rank + 1]] +xAll = rng.standard_normal(nCols) +yAll = rng.standard_normal(nRows) +x = xAll[colOff[rank]:colOff[rank + 1]] +y = yAll[rowOff[rank]:rowOff[rank + 1]] +# what rounding alone can make two evaluations of each product differ by (every rank has every entry) +absA = np.zeros((nRows, nCols)) +for (r, cols, vals) in entries: + absA[r, cols] = np.abs(vals) +eps = np.finfo(float).eps +multBound = eps * int((absA > 0).sum(axis=1).max()) * float((absA @ np.abs(xAll)).max()) +transBound = eps * int((absA > 0).sum(axis=0).max()) * float((absA.T @ np.abs(yAll)).max()) px, py = pm.createVecRight(), pm.createVecLeft() dx, dy = dm.createVecRight(), dm.createVecLeft() px.getArray()[:] = x; dx.getArray()[:] = x @@ -71,5 +87,6 @@ revErr = mpi.allreduce(float(np.max(np.abs(arrP[:n] - arrM[:n]), initial=0.0)), op=max) if rank == 0: print("RESULT mult=%.3e multTranspose=%.3e forward=%.3e forward_truth=%.3e reverse=%.3e" - % (multErr, transErr, fwdErr, fwdTruth, revErr)) + " mult_bound=%.3e multTranspose_bound=%.3e" + % (multErr, transErr, fwdErr, fwdTruth, revErr, multBound, transBound)) sys.stdout.flush() diff --git a/test/multilevel/test_without_petsc4py.py b/test/multilevel/test_without_petsc4py.py index 2638d72c77..0011518cb8 100644 --- a/test/multilevel/test_without_petsc4py.py +++ b/test/multilevel/test_without_petsc4py.py @@ -8,9 +8,12 @@ PETSc C library is still used by the graph partitioner, unless the partition is prescribed (test_partition_arrays.py runs multigrid with no PETSc at all). -* The building blocks agree with PETSc's exactly: DistMat's mult and - multTranspose, ParVec_mpi4py's forward insert and reverse add, on uneven - layouts with empty ranks. +* The building blocks agree with PETSc's, on uneven layouts with empty ranks: + ParVec_mpi4py's forward insert and reverse add exactly, DistMat's mult and + multTranspose to rounding. They agree exactly on the conda builds; the fleet's + pip PETSc on aarch64 (spark2) differs by 4.4e-16-8.9e-16, a fused + multiply-add, so the products are held to the rounding bound distmat_mpi.py + prints (about 1e-14 here) instead of to zero. * MGM run with petsc4py blocked (without_petsc4py.py) reproduces the PETSc path at every rank count, for nodal P1, P2, discontinuous P1 and hexahedral Q1. The PETSc path is in turn the serial cycle (test_parallel_mgm.py and friends), so @@ -62,8 +65,10 @@ def _launch(args, n_ranks, blocked, timeout=300): def test_building_blocks_agree_with_petsc(n_ranks): pytest.importorskip('petsc4py') got = _launch(['distmat_mpi.py', '40', '23'], n_ranks, blocked=False) - for key in ('mult', 'multTranspose', 'forward', 'forward_truth', 'reverse'): + for key in ('forward', 'forward_truth', 'reverse'): assert float(got[key]) == 0.0, got + for key in ('mult', 'multTranspose'): + assert float(got[key]) <= float(got[key + '_bound']), got CASES = { From cfe10f68fa8f2576b472000f8f79b6b03d651827 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:05:11 +0100 Subject: [PATCH 2/6] tests: a mumps marker, so a PETSc without MUMPS skips with a reason instead of failing 23 tests The fleet nightly's dev_pip prefix (pip-built PETSc, no MUMPS) failed 23 tests every night that its dev_pip_mumps prefix passes (2026-11-05, main 8f948a1c, ceds-model: dev_pip 32 failed, dev_pip_mumps 10, the difference exactly these 23). Each names MUMPS or needs the direct solve it stands for: explicit "linear_solver mumps, but this PETSc was built without MUMPS" or "Could not locate solver type mumps": test_adr_problem (pinned Taylor-Hood, periodic velocity-pressure, the superlu comparison's mumps case, the periodic solver comparison, the 2- and 3-rank periodic and advected-periodic runs, the th_mumps refusal), the 4 parallel DG velocity post-processing cases, the RANS3PF and CLSVOF rank-cut runs; the no-MUMPS fallback (linear_solver petsc, GMRES+ILU(0)): the pressure null space (adr_problem, pro CLI on two ranks: ILU fails on the empty pressure diagonal), pure advection across a rank cut (5 weak-flux classes: L2 inf); test_the_linear_solver_choice_follows_the_spec: an omitted solver on a singular operator picks MUMPS only when PETSc has it. @pytest.mark.mumps(reason) marks them; conftest probes PETSc once (in a subprocess, so petsc4py does not start PETSc in the test process) when a marked test is about to run, and skips it with "this PETSc was built without MUMPS, and ". Under GitHub Actions nothing is skipped: every CI leg's PETSc is conda-forge's, which has MUMPS, and a CI environment that lost it must fail rather than go green with 23 skips. The fallback failures are a known gap, not a test defect: proteus has no scalable no-MUMPS solver for singular and saddle-point systems yet (cekees, 2026-10-29: the nodes get MUMPS for the nightly, and the no-MUMPS option is still wanted). The marker's reason says so in the skip summary. checked here (chewbacca, PETSc with MUMPS): 23 collected by -m mumps; with the probe forced to "no": 23 skipped with their reasons; with GITHUB_ACTIONS=true and the probe forced to "no": they run (choice test: 1 passed); with the real probe: they run (2 passed of the 2 tried) Co-Authored-By: Claude Opus 5.5 --- test/conftest.py | 38 +++++++++++++++++++ .../test_velocity_postprocessing.py | 1 + test/test_adr_problem.py | 13 ++++++- test/test_pro_cli.py | 1 + test/test_rank_interface_flux.py | 2 + 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 64209535b3..509fc1a2f1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -24,6 +24,8 @@ def pytest_configure(config): """ import signal import threading + config.addinivalue_line('markers', 'mumps(reason): needs a PETSc built with MUMPS ' + '(skipped without it, except under GitHub Actions)') if (threading.current_thread() is threading.main_thread() and signal.getsignal(signal.SIGTERM) == signal.SIG_DFL): signal.signal(signal.SIGTERM, _end_launches_on_sigterm) @@ -80,6 +82,42 @@ def pytest_ignore_collect(collection_path, config): return None +# MUMPS. Every CI leg's PETSc (conda-forge) has it; a PETSc built without it (the fleet's +# pip-built dev_pip prefix, a hand-built PETSc) cannot run a test that names MUMPS or that needs +# the direct solve it stands for: the singular and saddle-point systems whose fallback, +# `linear_solver: petsc` with PETSc's default GMRES+ILU(0), fails or diverges (a known gap: +# proteus has no scalable no-MUMPS solver for them yet). Such a test carries +# @pytest.mark.mumps(reason) and is skipped there, with the reason in the summary, instead of +# failing 22 times a night. Under GitHub Actions nothing is skipped: a CI environment that lost +# MUMPS must fail, not go quietly green. The probe runs once, in a subprocess (importing petsc4py +# here would start PETSc before a test could give it its own options), and only when a marked test +# is about to run. +_MUMPS = [] + + +def petsc_has_mumps(): + """Whether the PETSc that petsc4py loads was built with MUMPS (probed once per process).""" + if not _MUMPS: + probe = ("import petsc4py; petsc4py.init([]); from petsc4py import PETSc; " + "print(int(PETSc.Sys.hasExternalPackage('mumps')))") + try: + out = subprocess.run([sys.executable, '-c', probe], capture_output=True, text=True, + timeout=120).stdout.strip() + _MUMPS.append(out.endswith('1')) + except (OSError, subprocess.SubprocessError): + _MUMPS.append(True) # cannot tell: run the test and let it say + return _MUMPS[0] + + +def pytest_runtest_setup(item): + marker = item.get_closest_marker('mumps') + if marker is None or os.environ.get('GITHUB_ACTIONS') == 'true': + return + if not petsc_has_mumps(): + why = marker.args[0] if marker.args else marker.kwargs.get('reason', 'it solves with MUMPS') + pytest.skip('this PETSc was built without MUMPS, and ' + why) + + def pytest_report_header(config): lines = [] if _MISSING: diff --git a/test/post_processing/test_velocity_postprocessing.py b/test/post_processing/test_velocity_postprocessing.py index 9d27e6cc3c..ed81ad8124 100644 --- a/test/post_processing/test_velocity_postprocessing.py +++ b/test/post_processing/test_velocity_postprocessing.py @@ -208,6 +208,7 @@ def test_a_dg_variant_balances_every_element(tmp_path, variant, order): @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("its parallel DG solve is LU through MUMPS (vpp_partition_mpi.py)") @pytest.mark.parametrize("variant,order,ranks", DG_PARALLEL, ids=["%s-P%d-%dranks" % case for case in DG_PARALLEL]) def test_a_dg_variant_in_parallel_is_the_serial_answer(tmp_path, variant, order, ranks): """Element partition, one layer of overlap: every owned element has its neighbours. 4e-12 measured.""" diff --git a/test/test_adr_problem.py b/test/test_adr_problem.py index 1ba9e80829..ef1e315d91 100644 --- a/test/test_adr_problem.py +++ b/test/test_adr_problem.py @@ -356,6 +356,7 @@ def test_the_emitted_p_file_carries_the_neumann_flux(tmp_path): CG1 = {"u": ("CG", 1)} +@pytest.mark.mumps("an omitted linear_solver on a singular operator picks MUMPS only when PETSc has it") def test_the_linear_solver_choice_follows_the_spec(): choice = ADRProblem.linear_solver_choice plain, singular = {}, {"up_to_constant": ["u"]} @@ -388,7 +389,7 @@ def test_a_singular_operator_is_not_handed_to_superlu(): @pytest.mark.parametrize("linear_solver, petsc_options", [ - ("mumps", None), + pytest.param("mumps", None, marks=pytest.mark.mumps("it names linear_solver mumps")), ("petsc", {"ksp_type": "cg", "pc_type": "jacobi"}), ("petsc", None), # PETSc's own defaults ]) @@ -723,6 +724,7 @@ def kovasznay_spec(tmp_path): return path +@pytest.mark.mumps("the pressure null space is solved through MUMPS; the fallback without it (GMRES+ILU(0)) fails") def test_a_pressure_up_to_a_constant_is_solved_in_the_null_space(kovasznay_spec, tmp_path): import numpy out = ymf_run(kovasznay_spec, "--outdir", tmp_path, "--emit-pn") @@ -757,6 +759,7 @@ def test_a_pressure_up_to_a_constant_is_solved_in_the_null_space(kovasznay_spec, assert all(numpy.array_equal(mine[k], recorded[k]) for k in recorded) +@pytest.mark.mumps("it names linear_solver mumps") def test_a_pinned_taylor_hood_pressure_solves_through_mumps(kovasznay_spec, tmp_path): """A kept pressure datum needs a diagonal the continuity equation lacks. @@ -832,6 +835,7 @@ def _errors_by_branch(stdout): return found +@pytest.mark.mumps("one of the linear solvers it compares is MUMPS") def test_a_periodic_problem_solves_the_same_through_each_linear_solver(tmp_path): """The serial periodic path under KSP_petsc4py. @@ -903,6 +907,7 @@ def test_a_periodic_problem_solves_the_same_through_each_linear_solver(tmp_path) """ +@pytest.mark.mumps("it names linear_solver mumps") def test_a_periodic_velocity_pressure_system_solves_through_mumps(tmp_path): """Mixed spaces, a pinned periodic pressure, and unequal free counts. @@ -955,6 +960,7 @@ def _recorded_ranks(archive): @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("it runs the spec's mumps branch") @pytest.mark.parametrize("ranks", [2, 3]) def test_a_periodic_problem_solves_on_several_ranks_as_on_one(tmp_path, ranks): """Proteus's parallel periodic numbering, on an element partition without @@ -983,6 +989,7 @@ def test_a_periodic_problem_solves_on_several_ranks_as_on_one(tmp_path, ranks): @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("it runs the spec's mumps branch") @pytest.mark.parametrize("ranks", [2, 3]) def test_an_advected_periodic_problem_solves_on_several_ranks_as_on_one(tmp_path, ranks): pytest.importorskip("ymf.symbolic") @@ -1001,6 +1008,7 @@ def test_an_advected_periodic_problem_solves_on_several_ranks_as_on_one(tmp_path @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("it runs the spec's th_mumps branch, which without MUMPS is refused for that instead") def test_what_the_parallel_periodic_path_cannot_solve_is_refused(tmp_path): pytest.importorskip("ymf.symbolic") pytest.importorskip("strictyaml") @@ -1092,7 +1100,8 @@ def test_advection_along_a_rank_interface_is_untouched(tmp_path): "Advection_DiagonalUpwind_Diffusion_IIPG_exterior", "Advection_DiagonalUpwind_Diffusion_SIPG_exterior", "Advection_DiagonalUpwind_Diffusion_NIPG_exterior"]) -@pytest.mark.parametrize("advonly", [0, 1]) +@pytest.mark.parametrize("advonly", [0, pytest.param(1, marks=pytest.mark.mumps( + "pure advection is solved through MUMPS when PETSc has it; the fallback (GMRES+ILU(0)) gives inf"))]) def test_weak_advective_flux_classes_agree_with_serial_across_a_rank_cut(flux, advonly, tmp_path): """The CG weak-boundary upwind classes need no rank-interface mask. diff --git a/test/test_pro_cli.py b/test/test_pro_cli.py index d000ca16c1..02650c4b92 100644 --- a/test/test_pro_cli.py +++ b/test/test_pro_cli.py @@ -1231,6 +1231,7 @@ def test_superlu_is_refused_on_two_ranks_before_computing(tmp_path): @needs_mpiexec +@pytest.mark.mumps("the pressure null space is solved through MUMPS; the fallback without it (GMRES+ILU(0)) fails" " in ILU: proteus has no scalable no-MUMPS solver for singular saddle-point systems yet") def test_a_pressure_null_space_runs_on_two_ranks(tmp_path): """Taylor-Hood (each rank's DOFs blocked by component) and SUPG/PSPG P1/P1 (interleaved, block size 3), the pressure fixed only up to a constant.""" diff --git a/test/test_rank_interface_flux.py b/test/test_rank_interface_flux.py index 0e0a55e3ac..78ab150ce3 100644 --- a/test/test_rank_interface_flux.py +++ b/test/test_rank_interface_flux.py @@ -191,6 +191,7 @@ def _rans3pf(ranks, cwd, **overrides): @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("its pressure and momentum solves are LU through MUMPS") def test_rans3pf_projection_scheme_on_two_ranks_gives_the_serial_answer(tmp_path): """Momentum flux on the rank cut: 2.4706e-03 on two ranks without the zero-flux entry, 2.4886e-03 with it and on one rank.""" @@ -224,6 +225,7 @@ def _clsvof(ranks, cwd, **overrides): @pytest.mark.skipif(MPIEXEC is None, reason="mpiexec not available") +@pytest.mark.mumps("its CLSVOF solve is LU through MUMPS") def test_clsvof_on_a_partition_with_overlap_gives_the_serial_answer_and_without_is_refused(tmp_path): """CLSVOF's lumped projections are not summed across ranks: a partition without overlap gave a 3e-3 relative error in the first step, so it is refused.""" From 3aab9aa84ae143a98a1d3b78b2eea0bd30bf8486 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:05:51 +0100 Subject: [PATCH 3/6] multilevel: pin the independent-partition controls to partitions that cannot nest test_independent_partitions_do_not_have_this_property[node-hex] failed on cekees/proteus#198's linux-64/py3.12 leg (run 42024054393, ceds-model-4): missing = 0, i.e. four independently partitioned levels of the 4x4x4 hex mesh nested. The control asked the graph partitioner for each level's cut and asserted it would not nest, so its outcome belonged to the partitioner. Measured, graph-partitioned independent levels, 4 ranks, node partitioning: ceds-model, CI's env (proteus-ci-main, Open MPI 5.0.11), main c09280cb, hex 200 runs: missing 1427-6207 (median 3731), offproc 0.35-0.96, never 0 chewbacca (conda MPICH), 30 runs each: hex missing 1585-6269, quad 59-640, tri 96-578, tet 1074-3910; offproc 0.24-0.96 (quad's 0.2368 is the old 0.25-floor failure) PETSC_OPTIONS=-mat_partitioning_type current: hex missing 0 offproc 0.200, quad missing 0 offproc 0.151, tri 26, tet 156 So the CI zero is not the tail of the random draws; the contiguous initial partition reproduces it exactly, but node-quad passed just before node-hex on the same xdist worker, so a leaked PETSC_OPTIONS does not explain it either. Not root-caused; the control no longer depends on it. pinned_partitions.py partitions every level independently by slabs across one coordinate axis that turns with the level (x, y, x / x, y, z), near-equal contiguous runs of nodes (or element barycenters) in that coordinate, ties by index, no empty rank. Consecutive levels are cut across different axes, so a fine node at the far end of its slab has its parents in another rank's slab, beyond one layer of overlap. level_consistent_mpi.py takes it as a fifth argument (`slabs`; `graph` stays the default) and space_transfer_mpi.py as PROTEUS_TEST_INDEPENDENT_PARTITIONS=slabs, so the refusal control (test_operators_without_level_consistency_are_refused_not_wrong, p2 and ncp1) is pinned too. The offproc floor of the control rises from 0.15 to 0.5. pinned, 4 ranks: missing tri 280 / 372 (node / element), tet 2005 / 6057, quad 330 / 168, hex 3262 / 860; node offproc 0.72, 0.70, 0.64, 0.67 refusal: p2 and ncp1 refuse ("not level-consistent") on 3 of 4 ranks test_level_consistent_partitioning.py: 26 passed; the 10 controls 50 times in a loop (-n 4): 50/50 passed; 26 passed with PETSC_OPTIONS=-mat_partitioning_type current in the environment Co-Authored-By: Claude Opus 5.5 --- test/multilevel/level_consistent_mpi.py | 10 ++- test/multilevel/pinned_partitions.py | 82 +++++++++++++++++++ test/multilevel/space_transfer_mpi.py | 7 +- .../test_level_consistent_partitioning.py | 26 +++--- 4 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 test/multilevel/pinned_partitions.py diff --git a/test/multilevel/level_consistent_mpi.py b/test/multilevel/level_consistent_mpi.py index 96827e124f..a1040daf90 100644 --- a/test/multilevel/level_consistent_mpi.py +++ b/test/multilevel/level_consistent_mpi.py @@ -11,7 +11,11 @@ owned by another rank, i.e. what PETSc has to communicate. * ``imbalance`` -- largest owned-node count over the mean, per level. -Usage: level_consistent_mpi.py KIND LEVELS CONSISTENT [node|element] (KIND tri, tet, quad or hex) +Usage: level_consistent_mpi.py KIND LEVELS CONSISTENT [node|element [graph|slabs]] (KIND tri, tet, quad or hex) + +With CONSISTENT 0, ``graph`` (the default) cuts every level with the graph partitioner, +a draw that changes from run to run and can nest by luck; ``slabs`` pins each level to +slabs across an axis that turns with the level (pinned_partitions.py), which do not nest. With element partitioning ``missing`` counts the fine elements this rank owns whose parent element is not in the coarse subdomain, and ``offproc`` is not measured. @@ -35,7 +39,11 @@ nLevels = int(sys.argv[2]) consistent = bool(int(sys.argv[3])) by = sys.argv[4] if len(sys.argv) > 4 else 'node' +independentOwners = sys.argv[5] if len(sys.argv) > 5 else 'graph' MeshTools.MultilevelMesh.levelConsistentPartitioning = consistent +if not consistent and independentOwners == 'slabs': + from pinned_partitions import pin_independent_levels + pin_independent_levels(2 if kind in ('tri', 'quad') else 3) part = (MeshTools.MeshParallelPartitioningTypes.node if by == 'node' else MeshTools.MeshParallelPartitioningTypes.element) if kind == 'tri': diff --git a/test/multilevel/pinned_partitions.py b/test/multilevel/pinned_partitions.py new file mode 100644 index 0000000000..9615a7d96a --- /dev/null +++ b/test/multilevel/pinned_partitions.py @@ -0,0 +1,82 @@ +"""Independent level partitions that are known not to nest; imported by the MPI helpers, not collected. + +The controls in test_level_consistent_partitioning.py show that a finer level which +does not inherit the coarser one's partition misses parents. Cut by the graph +partitioner, an independent level is whatever the partitioner gives that run: the cut +changes from run to run and from build to build, and so does what it shares with the +level above. One CI run found no missing parent at all on the 4-rank hex mesh, and +-mat_partitioning_type current (the initial, contiguous partition) gives exactly that. +Pinned partitions make the control independent of the partitioner and its options. + +``pin_independent_levels()`` replaces the graph partitioner for every level with +slabs across one coordinate axis, the axis turning with the level: level 0 is cut +across x, level 1 across y, level 2 across x again (z joins in 3D). Each rank owns a +near-equal, contiguous run of nodes (or elements, by barycenter) in the order of that +coordinate, so no rank is empty. Two consecutive levels are cut across different +axes, so a fine node at the far end of its slab has parents in another rank's slab, +beyond any one layer of overlap: the partitions do not nest on any of the test meshes, +whatever the build. + +``PROTEUS_TEST_INDEPENDENT_PARTITIONS=slabs`` selects them in the helpers that take +that switch; any other non-empty value keeps the graph partitioner. +""" +import os + +import numpy as np +from proteus import MeshTools + +__all__ = ['pin_independent_levels', 'slab_owners', 'requested'] + + +def requested(): + """Whether the environment asks for pinned (slab) independent partitions.""" + return os.environ.get('PROTEUS_TEST_INDEPENDENT_PARTITIONS', '') == 'slabs' + + +def slab_owners(points, axis, size): + """The owner of each point: ``size`` near-equal runs in the order of coordinate ``axis``. + + Ties are broken by index (a stable sort), so the owners depend only on the points. + """ + order = np.argsort(np.asarray(points)[:, axis], kind='stable') + owners = np.empty(len(order), dtype=np.intc) + starts = (np.arange(size + 1) * len(order)) // size + for r in range(size): + owners[order[starts[r]:starts[r + 1]]] = r + return owners + + +def pin_independent_levels(dim): + """Partition every level of the next multilevel meshes independently, by turning slabs. + + Patches Mesh.partitionMesh so that a call without prescribed owners gets slab owners + instead of the graph partitioner; the n-th such call (the n-th level) is cut across + axis n mod dim. Turns level-consistent partitioning off. Returns the list the + patched method appends each level's axis to. + """ + from proteus import Comm + MeshTools.MultilevelMesh.levelConsistentPartitioning = False + original = MeshTools.Mesh.partitionMesh + axes = [] + + def partitionMesh(self, nLayersOfOverlap=1, + parallelPartitioningType=MeshTools.MeshParallelPartitioningTypes.node, + nodeOwners=None, elementOwners=None, nodeWeights=None): + size = Comm.get().size() + if size > 1 and nodeOwners is None and elementOwners is None: + axis = len(axes) % dim + axes.append(axis) + nodes = np.asarray(self.nodeArray) + if parallelPartitioningType == MeshTools.MeshParallelPartitioningTypes.node: + nodeOwners = slab_owners(nodes, axis, size) + else: + elementNodes = np.asarray(self.elementNodesArray) + elementOwners = slab_owners(nodes[elementNodes].mean(axis=1), axis, size) + nodeWeights = None + return original(self, nLayersOfOverlap=nLayersOfOverlap, + parallelPartitioningType=parallelPartitioningType, + nodeOwners=nodeOwners, elementOwners=elementOwners, + nodeWeights=nodeWeights) + + MeshTools.Mesh.partitionMesh = partitionMesh + return axes diff --git a/test/multilevel/space_transfer_mpi.py b/test/multilevel/space_transfer_mpi.py index 8fc8c296e5..65a169a43c 100644 --- a/test/multilevel/space_transfer_mpi.py +++ b/test/multilevel/space_transfer_mpi.py @@ -24,9 +24,14 @@ rank, size = comm.rank(), comm.size() mpi = comm.comm.tompi4py() testroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -# a control: partition every level independently, as proteus used to +# a control: partition every level independently, as proteus used to; 'slabs' pins the +# levels to partitions that do not nest (pinned_partitions.py), anything else draws them +# from the graph partitioner if os.environ.get('PROTEUS_TEST_INDEPENDENT_PARTITIONS'): MeshTools.MultilevelMesh.levelConsistentPartitioning = False + from pinned_partitions import requested, pin_independent_levels + if requested(): + pin_independent_levels(2) spaceName = sys.argv[1] nLevels = int(sys.argv[2]) degree = int(sys.argv[3]) diff --git a/test/multilevel/test_level_consistent_partitioning.py b/test/multilevel/test_level_consistent_partitioning.py index 60f81a8d3a..fea1031b30 100644 --- a/test/multilevel/test_level_consistent_partitioning.py +++ b/test/multilevel/test_level_consistent_partitioning.py @@ -16,7 +16,10 @@ (``missing`` is zero), on simplices, quadrilaterals and hexahedra, for node and element partitioning; and the off-process share of the prolongation is small; * the control is real: without inheritance ``missing`` is large, so the zero - above is the partitioner's doing and not a property of the small meshes; + above is the partitioner's doing and not a property of the small meshes. The + control's independent levels are pinned (pinned_partitions.py: slabs across an + axis that turns with the level), not cut by the graph partitioner, whose cut is + a random draw that can nest by luck; * the element table the subdomain hierarchy relies on agrees with matching elements through their nodes, which does not depend on how the partitioner numbers anything; @@ -61,17 +64,16 @@ def _launch(helper, args, n_ranks, env_extra=None, check=True, timeout=300): return result -def _measure(kind, consistent, by, n_ranks, levels=3): - result = _launch('level_consistent_mpi.py', [kind, str(levels), str(int(consistent)), by], - n_ranks) +def _measure(kind, consistent, by, n_ranks, levels=3, independent='graph'): + result = _launch('level_consistent_mpi.py', + [kind, str(levels), str(int(consistent)), by, independent], n_ranks) line = [l for l in result.stdout.splitlines() if l.startswith('RESULT')][0] return dict(item.split('=') for item in line.split()[1:]) OFFPROC_BOUND = 0.25 -# the control: independent cuts are random, 0.24-0.99 over a few runs on a 4-rank quad mesh, so -# only a bound well below the worst luck can be asserted of them -INDEPENDENT_OFFPROC_FLOOR = 0.15 +# the control's pinned slabs put 0.64-0.72 of the prolongation off process on these meshes +INDEPENDENT_OFFPROC_FLOOR = 0.5 @pytest.mark.parametrize('kind', ['tri', 'tet', 'quad', 'hex']) @@ -90,19 +92,19 @@ def test_every_parent_of_an_owned_node_is_on_the_rank(kind, by, n_ranks): @pytest.mark.parametrize('kind', ['tri', 'tet', 'quad', 'hex']) @pytest.mark.parametrize('by', ['node', 'element']) def test_independent_partitions_do_not_have_this_property(kind, by): - got = _measure(kind, False, by, 4) + # Pinned, not drawn: the graph partitioner's independent cuts change from run to + # run, and one in a few dozen nests by luck (missing == 0 on hex, CI run 42024054393). + # The slabs turn axis from level to level, so they never nest, on any build. + got = _measure(kind, False, by, 4, independent='slabs') assert int(got['missing']) > 0, got if by == 'node': - # independent cuts of the finer levels vary from run to run (hex on 4 - # ranks: 0.40-0.99 over ten runs, quad 0.24-0.90, against a fixed ~0.12 inherited), - # so the control is only that they stay well above what inheritance gets assert float(got['offproc']) > INDEPENDENT_OFFPROC_FLOOR, got @pytest.mark.parametrize('space,degree', [('p2', 2), ('ncp1', 1)]) def test_operators_without_level_consistency_are_refused_not_wrong(space, degree): result = _launch('space_transfer_mpi.py', [space, '3', str(degree)], 4, - env_extra={'PROTEUS_TEST_INDEPENDENT_PARTITIONS': '1'}, + env_extra={'PROTEUS_TEST_INDEPENDENT_PARTITIONS': 'slabs'}, check=False) assert result.returncode != 0, result.stdout assert 'level-consistent' in result.stdout + result.stderr, result.stdout + result.stderr From 1383a586b7e99b8f8f49a9a8f078f697fa31bae1 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:12:53 +0100 Subject: [PATCH 4/6] conftest: put default_p/default_n/default_so back after each test module is collected Round 4 found the suite order-dependent: the 39 test files that import reload pass in one process forward and fail reversed. Bisected here by pairs (each earlier file, then the failing file, in one pytest process; 159 pairs on ceds-model, CI's env): six files make the others fail, and all six import a deck at module level that reloads proteus.default_p/default_n and sets attributes on them (matrix_constructor/test_mass_matrix_quad.py and test_mass_matrix_reference_simplex.py; post_processing/test_bdm_projection.py, test_bdm2_mesh8.py, test_bdm_3Dprojection.py, test_elementwise_flux.py: the BDM templates set n.triangleOptions = "Yp", p.nd, p.name, ...). Importing a test module is collecting it, which happens before any test runs and outside _restore_proteus_process_state, whose snapshot is taken after collection. So whatever a module set while it was imported stayed Proteus's defaults, and every deck collected after it star-imported them: the order of the files on the command line decided which. A hookwrapper on pytest_make_collect_report now snapshots the three modules before each pytest.Module is collected and restores them after (dicts and lists copied, so in-place edits are undone too). the 39 files in one process (ceds-model, proteus-ci-main py3.12, Open MPI 5), main c09280cb -> this commit: forward 166 passed -> 166 passed reversed 34 failed, 132 passed -> 166 passed random seed 1 32 failed -> 166 passed random seed 2 13 failed -> 166 passed random seed 3 14 failed -> 166 passed random seed 4 15 failed -> 166 passed (the 34: test_gauges' first row 0.5625 for 0.5, 13 poisson_2d cases, vortex2D x2, vortex3D, TADR x6 (IndexError), VOF x6, NCLS x4, CLSVOF+RANS2P) Co-Authored-By: Claude Opus 5.5 --- test/conftest.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 509fc1a2f1..b519bd0ffe 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -509,6 +509,51 @@ def opts(): return runtime.RunOptions() +_DEFAULTS_MODULES = ("proteus.default_p", "proteus.default_n", "proteus.default_so") + + +def _snapshot_defaults(): + """The namespaces of default_p/default_n/default_so, their dicts and lists copied (a + deck that sets n.femSpaces[0] changes the dict, not the name).""" + import importlib + snapshot = {} + for name in _DEFAULTS_MODULES: + namespace = vars(importlib.import_module(name)) + snapshot[name] = dict((key, (value.copy() if type(value) in (dict, list) else value)) + for key, value in namespace.items()) + return snapshot + + +def _restore_defaults(snapshot): + for name, saved in snapshot.items(): + namespace = vars(sys.modules[name]) + for key in list(namespace): + if key not in saved: + del namespace[key] + namespace.update(saved) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_make_collect_report(collector): + """Put default_p/default_n/default_so back after each test module is imported. + + Some test modules build their problem at import, and the decks they import + reload proteus.default_p/default_n and set attributes on them (the BDM and + mass-matrix templates: n.triangleOptions = "Yp", p.nd, ...). Importing is + collecting, which happens before any test and outside the per-test restore + below, so every test file collected after one of them star-imported its + settings as Proteus's defaults: run reversed, poisson_2d, TADR, VOF, NCLS, + vortex and CLSVOF+RANS2P failed. Forward, they were collected first and + passed. + """ + if not isinstance(collector, pytest.Module): + yield + return + snapshot = _snapshot_defaults() + yield + _restore_defaults(snapshot) + + @pytest.fixture(autouse=True) def _restore_proteus_process_state(): """Put proteus.iproteus.opts, the default_p/n/so modules, proteus.Context, From 1fa29745596fb529b322bcaaa4aff669f98ef78d Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:28:50 +0100 Subject: [PATCH 5/6] conftest: import proteus.defaults before the first defaults snapshot 1383a586 restores default_p/default_n/default_so after each test module is collected. proteus.defaults takes its lists of option names from dir(default_p) and dir(default_n) when it is first imported; when a polluting deck imported it first (in the middle of setting p.rdomain, ...), those names became "options", and after the restore Physics_base() raised KeyError: 'rdomain'. Found here (chewbacca): test_bdm_projection.py then test_poisson2d.py, collection error. On ceds-model's order runs test_gauges.py, collected first in the reversed order, had imported proteus.defaults cleanly, so 1383a586's numbers did not see it. The snapshot now imports proteus.defaults first. That imports petsc4py, which the first proteus test module collected does anyway; Comm.init hands Comm.argv to the options database when PETSc has already started. the 39 reload files in one process (ceds-model, proteus-ci-main): reversed, random seeds 1-8: 166 passed each (base c09280cb: 13-34 failed in reversed and seeds 1-4) each of the six polluting files collected first, the rest in list order: 166 passed each (1383a586 alone: collection error in that order) chewbacca: test_bdm_projection.py test_poisson2d.py test_tadr.py: 28 passed (1383a586: 1 error) Co-Authored-By: Claude Opus 5.5 --- test/conftest.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index b519bd0ffe..bc3e157cc5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -514,8 +514,17 @@ def opts(): def _snapshot_defaults(): """The namespaces of default_p/default_n/default_so, their dicts and lists copied (a - deck that sets n.femSpaces[0] changes the dict, not the name).""" + deck that sets n.femSpaces[0] changes the dict, not the name). + + proteus.defaults goes first: when it is first imported it takes its lists of + physics and numerics option names from dir(default_p) and dir(default_n), so + first imported by a deck in the middle of changing them, it would count that + deck's names (p.rdomain) as Proteus options, and Physics_base() would fail + once they were put back. (It imports petsc4py, as the first test module + collected would; Comm.init hands Comm.argv to the options database when PETSc + has already started.)""" import importlib + import proteus.defaults # noqa: F401 snapshot = {} for name in _DEFAULTS_MODULES: namespace = vars(importlib.import_module(name)) From 33ad04fe812c8fa1661414cb8c4a6334f13e0dd0 Mon Sep 17 00:00:00 2001 From: cekees <613654+cekees@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:23:36 +0100 Subject: [PATCH 6/6] Coverage floors: NCLS3P.py 76.2 -> 75.6 (round 5 moved 15 covered statements to Transport) The floor was recorded on main ce43b736 (33f30b1a): 522 of 685 statements, 76.2%. cekees/proteus#199 (C0 round 5, 76f3e125) replaced NCLS3P's own scatterDOFArray (15 statements, all covered) by the one on OneLevelTransport, so main c09280cb has 507 of 670, 75.67%: the same lines untested, a smaller file. Every pull request on c09280cb without #204 (whose staticmethod lines add covered statements to the file) is below the floor: this PR's coverage job (run 42113031204, NCLS3P the only file under its floor) and #212's both report NCLS3P 75.7%. The new floor is the measured value rounded down to a tenth. Same hunk as 7c2a5d52 on c0-r6, so the train merges both cleanly. Co-Authored-By: Claude Opus 5.5 --- .github/coverage/floors.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/coverage/floors.json b/.github/coverage/floors.json index c16b06851f..8e220e39ba 100644 --- a/.github/coverage/floors.json +++ b/.github/coverage/floors.json @@ -116,7 +116,7 @@ "proteus/mprans/MoveMesh.py": 86.4, "proteus/mprans/MoveMeshMonitor.py": 65.9, "proteus/mprans/NCLS.py": 75.4, - "proteus/mprans/NCLS3P.py": 76.2, + "proteus/mprans/NCLS3P.py": 75.6, "proteus/mprans/Pres.py": 76.1, "proteus/mprans/PresInc.py": 76.2, "proteus/mprans/PresInit.py": 23.2,