From fd0e5c386786d6f234648907cbab84fa5677e217 Mon Sep 17 00:00:00 2001 From: Paul Saxe Date: Thu, 22 Oct 2026 11:03:48 -0400 Subject: [PATCH] Parallel loop: cores per iteration automatic; resources marked 'automatic' (the new default) shares the job's cores among the iterations that run at once (iteration_cores); the default of 1 left 60 of 64 cores idle in job 5368. The resource settings are marked so seamm 2026.10.22's fingerprint ignores them and a resume survives changing them. Requires seamm>=2026.10.22. Co-Authored-By: Claude Opus 5.5 --- HISTORY.rst | 10 +++++++ docs/user_guide/index.rst | 11 +++++++- loop_step/loop.py | 55 ++++++++++++++++++++++++++++++++++-- loop_step/loop_parameters.py | 15 ++++++++-- requirements.txt | 2 +- tests/test_loop_step.py | 34 ++++++++++++++++++++++ 6 files changed, 120 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 494f186..c8030b7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,16 @@ ======= History ======= +2026.10.22 -- Bugfix: a parallel loop's iterations share the job's cores + * A parallel loop gave each iteration one core by default, so four replicas in a + 64-core job ran on four cores (job 5368). "Cores per iteration" now defaults to + automatic: the job's cores shared among the iterations that run at once, e.g. + 16 each for four replicas in a 64-core job. A number is used as given; + flowcharts saved with 1 keep it, so set them to automatic. + * The settings that say only how the iterations run -- iterations at once, cores, + memory and time per iteration, and where the calculations run -- can now be + changed before resuming the job (needs seamm 2026.10.22). + 2026.10.17.1 -- Warn about a fixed random seed in a loop * A step in a loop's body that uses a fixed random seed, such as LAMMPS's Velocities, gives every iteration the same random numbers: iterations that start diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 31a2a9a..3228461 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -111,7 +111,11 @@ Iterations at once How many iterations run at the same time on this machine. By default as many as the cores and memory allow, given the cores and memory per iteration. Cores per iteration - The cores each iteration's calculations use (default 1). + The cores each iteration's calculations use. The default, *automatic*, shares + the job's cores among the iterations that run at once: 16 each for 4 + replicas in a 64-core job, 8 each in a 32-core one. A number is used as + given. (Before 2026.10.22 the default was 1, which left most of the job's cores + idle.) Memory per iteration The most memory an iteration needs (default 2 GB), so that the iterations running together fit in the machine's memory. @@ -150,6 +154,11 @@ are not run again, finished ones are merged, and running ones resume from their own checkpoints. An iteration whose evaluator was killed while the job ran on (out of memory, say) is run again, resuming where it was, up to twice. +The settings above that say only how the iterations run -- iterations at once, +cores, memory and time per iteration, and where the calculations run -- can be +changed before resuming, e.g. to give the iterations more cores; changing any +other setting makes the job start again from the top. + Index ===== diff --git a/loop_step/loop.py b/loop_step/loop.py index ffee1f0..ccbe746 100644 --- a/loop_step/loop.py +++ b/loop_step/loop.py @@ -102,6 +102,43 @@ def emit(self, record): self.messages.append(record.getMessage()) +def iteration_cores( + setting, job_cores, n_iterations, at_once=None, job_memory=0, memory=None +): + """The cores of each iteration of a parallel loop. + + Parameters + ---------- + setting : int or str + "Cores per iteration": a number, or "automatic" to share the job's cores + among the iterations that run at once. + job_cores : int + The job's cores (its tasks). + n_iterations : int + The iterations to run. + at_once : int or None + "Iterations at once", or None for as many as fit. + job_memory, memory : int + The job's memory and each iteration's, bytes; 0 or None if not known. + + Returns + ------- + (int, int or None) + The cores per iteration, and how many iterations share the job's cores + (None when the number was given). A parallel loop gave each iteration one + core by default, so 4 replicas ran on 4 of a job's 64 cores (SEAMM job + 5368, 2026-10-22). + """ + if not (isinstance(setting, str) and setting.strip().lower() == "automatic"): + return max(1, int(setting)), None + shared = max(1, n_iterations) + if at_once: + shared = min(shared, int(at_once)) + if job_memory and memory: + shared = min(shared, max(1, int(job_memory // memory))) + return max(1, int(job_cores) // shared), shared + + def _yes(value): """A boolean parameter's value, as text or as a bool.""" if isinstance(value, str): @@ -946,7 +983,6 @@ def _run_parallel(self, P, context, checkpointer, state): self._merge_iteration_files(pending, loop_directory, job_directory) # The resources of each iteration - cores = max(1, int(P["cores per iteration"])) memory = _in_units(P["memory per iteration"], "byte") walltime = _in_units(P["time per iteration"], "s") at_once = P["iterations at once"] @@ -968,6 +1004,15 @@ def _run_parallel(self, P, context, checkpointer, state): stopped = tuple(state["stopped"]) last = length if stopped is None else 0 + cores, shared = iteration_cores( + P["cores per iteration"], + pool.cores, + max(0, last - state["next"] + 1), + at_once, + pool.memory, + memory, + ) + # Set each iteration up, write its snapshot and checkpoint, add its task keys = {} for k in range(state["next"], last + 1): @@ -1029,7 +1074,13 @@ def _run_parallel(self, P, context, checkpointer, state): printer.important( __( f"Running {n} iterations in parallel, each with {cores} cores " - f"and {P['memory per iteration']} of memory.", + f"and {P['memory per iteration']} of memory" + + ( + f": the job's {pool.cores} cores shared among the " + f"{shared} that run at once." + if shared + else "." + ), indent=self.indent + 4 * " ", ) ) diff --git a/loop_step/loop_parameters.py b/loop_step/loop_parameters.py index 8040e89..c3b9922 100644 --- a/loop_step/loop_parameters.py +++ b/loop_step/loop_parameters.py @@ -222,6 +222,7 @@ class LoopParameters(seamm.Parameters): ), }, "iterations at once": { + "resource": True, "default": "as many as fit", "kind": "integer", "default_units": "", @@ -235,15 +236,21 @@ class LoopParameters(seamm.Parameters): ), }, "cores per iteration": { - "default": 1, + "resource": True, + "default": "automatic", "kind": "integer", "default_units": "", - "enumeration": tuple(), + "enumeration": ("automatic",), "format_string": "", "description": "Cores per iteration:", - "help_text": "The cores each iteration uses for its calculations.", + "help_text": ( + "The cores each iteration uses for its calculations. 'automatic' " + "shares the job's cores among the iterations that run at once, " + "e.g. 16 each for 4 iterations in a 64-core job." + ), }, "memory per iteration": { + "resource": True, "default": 2.0, "kind": "float", "default_units": "GB", @@ -253,6 +260,7 @@ class LoopParameters(seamm.Parameters): "help_text": "The memory each iteration needs at most.", }, "time per iteration": { + "resource": True, "default": "not given", "kind": "float", "default_units": "h", @@ -278,6 +286,7 @@ class LoopParameters(seamm.Parameters): ), }, "placement": { + "resource": True, "default": "inline", "kind": "enumeration", "default_units": "", diff --git a/requirements.txt b/requirements.txt index a052f61..576c1a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ molsystem>=2026.10.17 psutil -seamm>=2026.10.17 +seamm>=2026.10.22 seamm-exec>=2026.10.5 diff --git a/tests/test_loop_step.py b/tests/test_loop_step.py index f2cbf1d..a97b78a 100644 --- a/tests/test_loop_step.py +++ b/tests/test_loop_step.py @@ -21,3 +21,37 @@ def test_content(response): """Sample pytest test function with the pytest fixture as an argument.""" # from bs4 import BeautifulSoup # assert 'GitHub' in BeautifulSoup(response.content).title.string + + +def test_iteration_cores(): + """'automatic' shares the job's cores among the iterations that run at once; + a number is used as given (SEAMM job 5368, 2026-10-22: 4 replicas ran on 4 of + 64 cores, one each, by default).""" + from loop_step.loop import iteration_cores + + assert iteration_cores("automatic", 64, 4) == (16, 4) + assert iteration_cores("automatic", 32, 4) == (8, 4) + # limited by 'iterations at once' or by memory + assert iteration_cores("automatic", 64, 8, at_once=2) == (32, 2) + assert iteration_cores("automatic", 64, 8, job_memory=8e9, memory=2e9) == (16, 4) + # more iterations than cores: one each + assert iteration_cores("automatic", 4, 10) == (1, 10) + assert iteration_cores(8, 64, 4) == (8, None) + assert iteration_cores("3", 64, 4) == (3, None) + + +def test_resource_parameters_are_marked(): + """How the iterations run, not what they compute: a resume survives changing + them (seamm's checkpoint fingerprint leaves them out).""" + from loop_step import LoopParameters + + P = LoopParameters() + for key in ( + "cores per iteration", + "memory per iteration", + "time per iteration", + "iterations at once", + "placement", + ): + assert P[key]._data.get("resource") is True, key + assert P["cores per iteration"].default == "automatic"