From db8517775c1d3df326bc42e296fee507fc6d3344 Mon Sep 17 00:00:00 2001 From: Matthew Keeler Date: Wed, 30 Sep 2026 13:54:26 +0200 Subject: [PATCH] Follow EngiBench naming every volume target volfrac EngiBench #280 and #281 rename the volume target to volfrac in thermoelastic2d and heatconduction2d and move both to new dataset repos holding the same rows. The committed specs named the old condition and dataset, so they no longer resolved. - Freeze thermoelastic2d/v3 and heatconduction2d/v3. Each is its v2 with the version, volume condition, dataset, digest and notes changed. Every metric, tolerance and seed is the same. - Delete the old versions of those two specs. Git history keeps them. - With no version given, load the newest spec committed for the problem, sorted numerically. This replaces a hardcoded v2 here and v1 in the CLI. - Pin CI to the EngiBench commit that has both renames, and assert both new dataset ids. - Rename the old condition names in test fixtures and the contributor guide. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/lint.yaml | 13 +++--- CONTRIBUTING_A_MODEL.md | 2 +- engiopt/evaluate.py | 4 +- engiopt/evaluation/evaluator.py | 2 +- engiopt/evaluation/spec.py | 7 +++- engiopt/specs/heatconduction2d/v1.json | 35 ---------------- .../heatconduction2d/{v2.json => v3.json} | 18 ++++----- engiopt/specs/thermoelastic2d/v1.json | 40 ------------------- .../thermoelastic2d/{v2.json => v3.json} | 18 ++++----- tests/test_checkpoint_packages.py | 4 +- tests/test_condition_schema.py | 4 +- tests/test_metrics_suite.py | 8 ++-- tests/test_specs.py | 24 +++++------ 13 files changed, 54 insertions(+), 125 deletions(-) delete mode 100644 engiopt/specs/heatconduction2d/v1.json rename engiopt/specs/heatconduction2d/{v2.json => v3.json} (56%) delete mode 100644 engiopt/specs/thermoelastic2d/v1.json rename engiopt/specs/thermoelastic2d/{v2.json => v3.json} (55%) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 77a7b853..f7025b04 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -39,18 +39,19 @@ jobs: python-version: "3.12" - uses: actions/checkout@v4 - run: pip install -e ".[testing]" - # The committed eval specs are frozen against the current EngiBench, whose - # photonics2d and thermoelastic2d read the v1 datasets. The newest release - # on PyPI (0.2.0) still points those two at v0, so the PyPI build draws - # different conditions and no spec could reproduce itself. + # The committed eval specs are frozen against the pinned EngiBench, where every + # volume target is named volfrac: thermoelastic2d reads thermoelastic_2d_v2 and + # heatconduction2d reads heat_conduction_2d_v1. The newest release on PyPI + # (0.2.0) predates that, so the PyPI build draws different conditions and no + # spec could reproduce itself. # # --force-reinstall is required, not decorative: the git build reports the # same version string (0.2.0) as the PyPI one, so a plain install is a # no-op and the specs would be verified against the wrong EngiBench. # --no-deps keeps it from re-resolving the whole stack. - - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@0a028c03d02b266cb4394968c386e45781f4605b" + - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@27055ae4248f0594a71e63950c7336462e7decec" # Fail loudly if the pin silently stopped taking effect. - - run: python -c "import engibench.problems.thermoelastic2d.v0 as m; assert m.ThermoElastic2D.dataset_id.endswith('_v1'), m.ThermoElastic2D.dataset_id" + - run: python -c "import engibench.problems.thermoelastic2d.v0 as t, engibench.problems.heatconduction2d.v0 as h; assert t.ThermoElastic2D.dataset_id.endswith('_v2'), t.ThermoElastic2D.dataset_id; assert h.HeatConduction2D.dataset_id.endswith('_v1'), h.HeatConduction2D.dataset_id" # Every committed spec must still resolve: same problem definition, same # pinned dataset revision, same digest. - run: pytest tests/ -v diff --git a/CONTRIBUTING_A_MODEL.md b/CONTRIBUTING_A_MODEL.md index 0f0e78e5..6ace84eb 100644 --- a/CONTRIBUTING_A_MODEL.md +++ b/CONTRIBUTING_A_MODEL.md @@ -67,7 +67,7 @@ solver settings that are not dataset columns at all. Size your network from the ```python from engiopt.transforms import condition_keys -cond_keys = condition_keys(problem) # e.g. ("volume_fraction_target", "rmin", "weight") +cond_keys = condition_keys(problem) # e.g. ("volfrac", "rmin", "weight") n_conds = len(cond_keys) ``` diff --git a/engiopt/evaluate.py b/engiopt/evaluate.py index 8a54f97a..748a56e8 100644 --- a/engiopt/evaluate.py +++ b/engiopt/evaluate.py @@ -64,7 +64,7 @@ class Args: cgan_cnn_2d:023dd1fb gan_cnn_2d:06d9a9a1` asks for exactly the two that exist.""" spec: str | None = None - """Eval spec reference, e.g. `beams2d/v1`. Defaults to `/v1`.""" + """Eval spec reference, e.g. `beams2d/v2`. Defaults to the newest spec for the problem.""" metrics: tuple[str, ...] = () """Metric names; defaults to the spec's list.""" include_expensive: bool = False @@ -341,7 +341,7 @@ def main(args: Args) -> int: if args.list_generators or args.list_metrics: return 0 - spec = args.spec or f"{args.problem_id}/v1" + spec = args.spec or args.problem_id # no version: the newest spec for the problem evaluator = Evaluator.for_problem(args.problem_id, spec=spec) print(f"Problem {args.problem_id} | spec {evaluator.spec.version} | n={evaluator.spec.n_samples}") diff --git a/engiopt/evaluation/evaluator.py b/engiopt/evaluation/evaluator.py index 927f8090..24095139 100644 --- a/engiopt/evaluation/evaluator.py +++ b/engiopt/evaluation/evaluator.py @@ -128,7 +128,7 @@ def for_problem( Args: problem_id: EngiBench problem registry key. spec: `"/"`, an `EvalSpec`, or None to load - `"/v1"`. + the newest spec committed for the problem. device: Torch device; auto-selected when omitted. registry: Metric registry override, useful in tests. """ diff --git a/engiopt/evaluation/spec.py b/engiopt/evaluation/spec.py index 4af8e720..a39c017d 100644 --- a/engiopt/evaluation/spec.py +++ b/engiopt/evaluation/spec.py @@ -187,12 +187,15 @@ def __post_init__(self) -> None: @classmethod def load(cls, reference: str, *, root: Path | None = None) -> EvalSpec: - """Load a spec from `"/"` or a path to a JSON file.""" + """Load a spec from `"/"`, `""` for the newest version, or a JSON path.""" path = Path(reference) if path.suffix == ".json" and path.exists(): return cls(**json.loads(path.read_text())) problem_id, _, version = reference.partition("/") - version = version or "v2" + if not version: + # No version given: use the newest spec committed for this problem. + committed = sorted((root or SPEC_ROOT).glob(f"{problem_id}/v*.json"), key=lambda p: int(p.stem[1:])) + version = committed[-1].stem if committed else "v1" spec_path = (root or SPEC_ROOT) / problem_id / f"{version}.json" if not spec_path.exists(): raise FileNotFoundError( diff --git a/engiopt/specs/heatconduction2d/v1.json b/engiopt/specs/heatconduction2d/v1.json deleted file mode 100644 index a0221ecd..00000000 --- a/engiopt/specs/heatconduction2d/v1.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "condition_digest": "40030d1b2482e003", - "condition_seed": 1, - "dataset_id": "IDEALLab/heat_conduction_2d_v0", - "dataset_revision": "9f07c1e70d244f783e34a71c96db621173871dc1", - "engibench_version": "0.2.0+0a028c03d02b", - "metrics": [ - "mmd", - "dpp", - "novelty", - "cond_sens", - "viol", - "iog", - "cog", - "fog" - ], - "n_samples": 50, - "notes": "Feasibility = EngiBench constraints plus the volume budget.", - "objective_weight_condition": null, - "objective_weights": null, - "problem_conditions": [ - "volume", - "length" - ], - "problem_id": "heatconduction2d", - "required_seeds": [ - 1, - 2, - 3 - ], - "sigma": 10.0, - "version": "v1", - "volfrac_tol": 0.01, - "volume_condition": "volume" -} diff --git a/engiopt/specs/heatconduction2d/v2.json b/engiopt/specs/heatconduction2d/v3.json similarity index 56% rename from engiopt/specs/heatconduction2d/v2.json rename to engiopt/specs/heatconduction2d/v3.json index bdfed215..8fc45e9d 100644 --- a/engiopt/specs/heatconduction2d/v2.json +++ b/engiopt/specs/heatconduction2d/v3.json @@ -1,9 +1,10 @@ { - "condition_digest": "40030d1b2482e003", + "aggregation": "mean", + "condition_digest": "aa13508d396239d5", "condition_seed": 1, - "dataset_id": "IDEALLab/heat_conduction_2d_v0", - "dataset_revision": "9f07c1e70d244f783e34a71c96db621173871dc1", - "engibench_version": "0.2.0+0a028c03d02b", + "dataset_id": "IDEALLab/heat_conduction_2d_v1", + "dataset_revision": "9277ee7476626250e5c9f9ed12e0dd21d181b532", + "engibench_version": "0.2.0+27055ae4248f", "metrics": [ "mmd", "coverage", @@ -25,11 +26,11 @@ "reaches_reference_rate" ], "n_samples": 50, - "notes": "Feasibility = EngiBench constraints plus the volume budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.", + "notes": "Feasibility = EngiBench constraints plus the volume budget. v3: v2 on heat_conduction_2d_v1, where the volume target is named volfrac. Same rows, same draws, same metrics as v2.", "objective_weight_condition": null, "objective_weights": null, "problem_conditions": [ - "volume", + "volfrac", "length" ], "problem_id": "heatconduction2d", @@ -39,8 +40,7 @@ 3 ], "sigma": null, - "version": "v2", + "version": "v3", "volfrac_tol": 0.01, - "volume_condition": "volume", - "aggregation": "mean" + "volume_condition": "volfrac" } diff --git a/engiopt/specs/thermoelastic2d/v1.json b/engiopt/specs/thermoelastic2d/v1.json deleted file mode 100644 index 0271b3d6..00000000 --- a/engiopt/specs/thermoelastic2d/v1.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "condition_digest": "818d93c2a2d5ac25", - "condition_seed": 1, - "dataset_id": "IDEALLab/thermoelastic_2d_v1", - "dataset_revision": "7d5b4a694bd1a8f3f25fd745ac9f669764e29ea5", - "engibench_version": "0.2.0+0a028c03d02b", - "metrics": [ - "mmd", - "dpp", - "novelty", - "cond_sens", - "viol", - "iog", - "cog", - "fog" - ], - "n_samples": 50, - "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volume_fraction_target budget.", - "objective_weight_condition": "weight", - "objective_weights": null, - "problem_conditions": [ - "fixed_elements", - "force_elements_x", - "force_elements_y", - "heatsink_elements", - "volume_fraction_target", - "rmin", - "weight" - ], - "problem_id": "thermoelastic2d", - "required_seeds": [ - 1, - 2, - 3 - ], - "sigma": 10.0, - "version": "v1", - "volfrac_tol": 0.01, - "volume_condition": "volume_fraction_target" -} diff --git a/engiopt/specs/thermoelastic2d/v2.json b/engiopt/specs/thermoelastic2d/v3.json similarity index 55% rename from engiopt/specs/thermoelastic2d/v2.json rename to engiopt/specs/thermoelastic2d/v3.json index 977cbd97..a9c1bab8 100644 --- a/engiopt/specs/thermoelastic2d/v2.json +++ b/engiopt/specs/thermoelastic2d/v3.json @@ -1,9 +1,10 @@ { - "condition_digest": "818d93c2a2d5ac25", + "aggregation": "mean", + "condition_digest": "3665bcd7e957e3e6", "condition_seed": 1, - "dataset_id": "IDEALLab/thermoelastic_2d_v1", - "dataset_revision": "7d5b4a694bd1a8f3f25fd745ac9f669764e29ea5", - "engibench_version": "0.2.0+0a028c03d02b", + "dataset_id": "IDEALLab/thermoelastic_2d_v2", + "dataset_revision": "cead6df28c96dabb53dd89714c779dff24bcdf49", + "engibench_version": "0.2.0+27055ae4248f", "metrics": [ "mmd", "coverage", @@ -25,7 +26,7 @@ "reaches_reference_rate" ], "n_samples": 50, - "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volume_fraction_target budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.", + "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volfrac budget. v3: v2 on thermoelastic_2d_v2, where the volume target is named volfrac and the volume-fraction error is no longer an objective. Same rows, same draws, same metrics as v2.", "objective_weight_condition": "weight", "objective_weights": null, "problem_conditions": [ @@ -33,7 +34,7 @@ "force_elements_x", "force_elements_y", "heatsink_elements", - "volume_fraction_target", + "volfrac", "rmin", "weight" ], @@ -44,8 +45,7 @@ 3 ], "sigma": null, - "version": "v2", + "version": "v3", "volfrac_tol": 0.01, - "volume_condition": "volume_fraction_target", - "aggregation": "mean" + "volume_condition": "volfrac" } diff --git a/tests/test_checkpoint_packages.py b/tests/test_checkpoint_packages.py index c55ba65d..f53b468e 100644 --- a/tests/test_checkpoint_packages.py +++ b/tests/test_checkpoint_packages.py @@ -214,10 +214,10 @@ def fake_upload(**kwargs: Any) -> str: seed=1, checkpoint_files={}, run_config={}, - condition_keys=["volume_fraction_target", "rmin", "weight"], + condition_keys=["volfrac", "rmin", "weight"], ) - assert uploads[0]["condition_keys"] == ["volume_fraction_target", "rmin", "weight"] + assert uploads[0]["condition_keys"] == ["volfrac", "rmin", "weight"] def test_recorded_schema_survives_the_problem_gaining_a_condition(tmp_path: Path) -> None: diff --git a/tests/test_condition_schema.py b/tests/test_condition_schema.py index f18ac209..8f667d68 100644 --- a/tests/test_condition_schema.py +++ b/tests/test_condition_schema.py @@ -26,11 +26,11 @@ "force_elements_x", "force_elements_y", "heatsink_elements", - "volume_fraction_target", + "volfrac", "rmin", "weight", ) -THERMOELASTIC_SCALARS = ("volume_fraction_target", "rmin", "weight") +THERMOELASTIC_SCALARS = ("volfrac", "rmin", "weight") class _EchoGenerator(Generator): diff --git a/tests/test_metrics_suite.py b/tests/test_metrics_suite.py index 603b03f7..95336185 100644 --- a/tests/test_metrics_suite.py +++ b/tests/test_metrics_suite.py @@ -261,12 +261,12 @@ def test_the_default_spec_metrics_are_all_registered() -> None: @pytest.mark.parametrize("problem_id", ["beams2d", "heatconduction2d", "photonics2d", "thermoelastic2d"]) -def test_every_v2_spec_names_only_registered_metrics(problem_id: str) -> None: - spec = EvalSpec.load(f"{problem_id}/v2") - assert spec.version == "v2" +def test_every_current_spec_names_only_registered_metrics(problem_id: str) -> None: + spec = EvalSpec.load(problem_id) assert spec.aggregation == "mean" assert set(spec.metrics) <= set(METRICS) -def test_the_default_spec_is_v2() -> None: +def test_the_default_spec_is_the_newest_committed_for_the_problem() -> None: assert EvalSpec.load("beams2d").version == "v2" + assert EvalSpec.load("thermoelastic2d").version == "v3" diff --git a/tests/test_specs.py b/tests/test_specs.py index b2bf4974..d5a1c94e 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -25,12 +25,13 @@ SPEC_IDS = [f"{path.parent.name}/{path.stem}" for path in SPEC_PATHS] CURRENT_PATHS = [ - max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: p.stem) + max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: int(p.stem[1:])) for problem in sorted({p.parent.name for p in SPEC_PATHS}) ] CURRENT_IDS = [f"{path.parent.name}/{path.stem}" for path in CURRENT_PATHS] -"""The newest spec per problem. Older versions stay committed so published rows can -be read under the protocol that produced them, and may name metrics since retired.""" +"""The newest spec per problem. Older versions stay committed only while the pinned +EngiBench still reproduces them. When EngiBench redefines a problem, its old spec +versions are deleted and git history keeps them.""" class _FakeConditions: @@ -108,7 +109,7 @@ def test_digest_changes_when_a_condition_is_renamed() -> None: indices = np.array([0, 1, 2]) designs = np.zeros((3, 4)) before = _digest(indices, _FakeConditions({"volfrac": [0.3, 0.4, 0.5]}), designs) - after = _digest(indices, _FakeConditions({"volume": [0.3, 0.4, 0.5]}), designs) + after = _digest(indices, _FakeConditions({"target_volume": [0.3, 0.4, 0.5]}), designs) assert before != after @@ -169,22 +170,21 @@ def test_a_differently_defined_problem_reports_itself() -> None: """ class _Problem: - conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "weight"] - dataset_id = "IDEALLab/thermoelastic_2d_v0" + conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "forcedist"] + dataset_id = "IDEALLab/beams_2d_50_100_v1" spec = EvalSpec( - problem_id="thermoelastic2d", - problem_conditions=("volume_fraction_target", "rmin", "weight"), - dataset_id="IDEALLab/thermoelastic_2d_v1", + problem_id="beams2d", + problem_conditions=("volfrac", "rmin"), + dataset_id="IDEALLab/beams_2d_50_100_v0", engibench_version="0.2.0", ) with pytest.raises(ProblemDefinitionMismatchError) as caught: spec.check_problem_definition(_Problem()) message = str(caught.value) - assert "volume_fraction_target" in message - assert "volfrac" in message - assert "thermoelastic_2d_v0" in message + assert "forcedist" in message + assert "beams_2d_50_100_v1" in message def test_a_matching_problem_definition_passes() -> None: