Follow EngiBench naming every volume target volfrac

.github/workflows/lint.yaml

@@ -39,18 +39,19 @@ jobs:
           python-version: "3.12"
       - uses: actions/checkout@v4
       - run: pip install -e ".[testing]"
-      # The committed eval specs are frozen against the current EngiBench, whose
-      # photonics2d and thermoelastic2d read the v1 datasets. The newest release
-      # on PyPI (0.2.0) still points those two at v0, so the PyPI build draws
-      # different conditions and no spec could reproduce itself.
+      # The committed eval specs are frozen against the pinned EngiBench, where every
+      # volume target is named volfrac: thermoelastic2d reads thermoelastic_2d_v2 and
+      # heatconduction2d reads heat_conduction_2d_v1. The newest release on PyPI
+      # (0.2.0) predates that, so the PyPI build draws different conditions and no
+      # spec could reproduce itself.
       #
       # --force-reinstall is required, not decorative: the git build reports the
       # same version string (0.2.0) as the PyPI one, so a plain install is a
       # no-op and the specs would be verified against the wrong EngiBench.
       # --no-deps keeps it from re-resolving the whole stack.
-      - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@0a028c03d02b266cb4394968c386e45781f4605b"
+      - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@27055ae4248f0594a71e63950c7336462e7decec"
       # Fail loudly if the pin silently stopped taking effect.
-      - run: python -c "import engibench.problems.thermoelastic2d.v0 as m; assert m.ThermoElastic2D.dataset_id.endswith('_v1'), m.ThermoElastic2D.dataset_id"
+      - run: python -c "import engibench.problems.thermoelastic2d.v0 as t, engibench.problems.heatconduction2d.v0 as h; assert t.ThermoElastic2D.dataset_id.endswith('_v2'), t.ThermoElastic2D.dataset_id; assert h.HeatConduction2D.dataset_id.endswith('_v1'), h.HeatConduction2D.dataset_id"
       # Every committed spec must still resolve: same problem definition, same
       # pinned dataset revision, same digest.
       - run: pytest tests/ -v

CONTRIBUTING_A_MODEL.md

@@ -67,7 +67,7 @@ solver settings that are not dataset columns at all. Size your network from the
 ```python
 from engiopt.transforms import condition_keys
 
-cond_keys = condition_keys(problem)  # e.g. ("volume_fraction_target", "rmin", "weight")
+cond_keys = condition_keys(problem)  # e.g. ("volfrac", "rmin", "weight")
 n_conds = len(cond_keys)
 ```
 

engiopt/evaluate.py

@@ -64,7 +64,7 @@ class Args:
     cgan_cnn_2d:023dd1fb gan_cnn_2d:06d9a9a1` asks for exactly the two that
     exist."""
     spec: str | None = None
-    """Eval spec reference, e.g. `beams2d/v1`. Defaults to `<problem_id>/v1`."""
+    """Eval spec reference, e.g. `beams2d/v2`. Defaults to the newest spec for the problem."""
     metrics: tuple[str, ...] = ()
     """Metric names; defaults to the spec's list."""
     include_expensive: bool = False
@@ -341,7 +341,7 @@ def main(args: Args) -> int:
     if args.list_generators or args.list_metrics:
         return 0
 
-    spec = args.spec or f"{args.problem_id}/v1"
+    spec = args.spec or args.problem_id  # no version: the newest spec for the problem
     evaluator = Evaluator.for_problem(args.problem_id, spec=spec)
     print(f"Problem {args.problem_id} | spec {evaluator.spec.version} | n={evaluator.spec.n_samples}")
 

engiopt/evaluation/evaluator.py

@@ -128,7 +128,7 @@ def for_problem(
         Args:
             problem_id: EngiBench problem registry key.
             spec: `"<problem_id>/<version>"`, an `EvalSpec`, or None to load
-                `"<problem_id>/v1"`.
+                the newest spec committed for the problem.
             device: Torch device; auto-selected when omitted.
             registry: Metric registry override, useful in tests.
         """

engiopt/evaluation/spec.py

@@ -187,12 +187,15 @@ def __post_init__(self) -> None:
 
     @classmethod
     def load(cls, reference: str, *, root: Path | None = None) -> EvalSpec:
-        """Load a spec from `"<problem_id>/<version>"` or a path to a JSON file."""
+        """Load a spec from `"<problem_id>/<version>"`, `"<problem_id>"` for the newest version, or a JSON path."""
         path = Path(reference)
         if path.suffix == ".json" and path.exists():
             return cls(**json.loads(path.read_text()))
         problem_id, _, version = reference.partition("/")
-        version = version or "v2"
+        if not version:
+            # No version given: use the newest spec committed for this problem.
+            committed = sorted((root or SPEC_ROOT).glob(f"{problem_id}/v*.json"), key=lambda p: int(p.stem[1:]))
+            version = committed[-1].stem if committed else "v1"
         spec_path = (root or SPEC_ROOT) / problem_id / f"{version}.json"
         if not spec_path.exists():
             raise FileNotFoundError(

engiopt/specs/heatconduction2d/v2.json → engiopt/specs/heatconduction2d/v3.json

@@ -1,9 +1,10 @@
 {
-  "condition_digest": "40030d1b2482e003",
+  "aggregation": "mean",
+  "condition_digest": "aa13508d396239d5",
   "condition_seed": 1,
-  "dataset_id": "IDEALLab/heat_conduction_2d_v0",
-  "dataset_revision": "9f07c1e70d244f783e34a71c96db621173871dc1",
-  "engibench_version": "0.2.0+0a028c03d02b",
+  "dataset_id": "IDEALLab/heat_conduction_2d_v1",
+  "dataset_revision": "9277ee7476626250e5c9f9ed12e0dd21d181b532",
+  "engibench_version": "0.2.0+27055ae4248f",
   "metrics": [
     "mmd",
     "coverage",
@@ -25,11 +26,11 @@
     "reaches_reference_rate"
   ],
   "n_samples": 50,
-  "notes": "Feasibility = EngiBench constraints plus the volume budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.",
+  "notes": "Feasibility = EngiBench constraints plus the volume budget. v3: v2 on heat_conduction_2d_v1, where the volume target is named volfrac. Same rows, same draws, same metrics as v2.",
   "objective_weight_condition": null,
   "objective_weights": null,
   "problem_conditions": [
-    "volume",
+    "volfrac",
     "length"
   ],
   "problem_id": "heatconduction2d",
@@ -39,8 +40,7 @@
     3
   ],
   "sigma": null,
-  "version": "v2",
+  "version": "v3",
   "volfrac_tol": 0.01,
-  "volume_condition": "volume",
-  "aggregation": "mean"
+  "volume_condition": "volfrac"
 }

engiopt/specs/thermoelastic2d/v2.json → engiopt/specs/thermoelastic2d/v3.json

@@ -1,9 +1,10 @@
 {
-  "condition_digest": "818d93c2a2d5ac25",
+  "aggregation": "mean",
+  "condition_digest": "3665bcd7e957e3e6",
   "condition_seed": 1,
-  "dataset_id": "IDEALLab/thermoelastic_2d_v1",
-  "dataset_revision": "7d5b4a694bd1a8f3f25fd745ac9f669764e29ea5",
-  "engibench_version": "0.2.0+0a028c03d02b",
+  "dataset_id": "IDEALLab/thermoelastic_2d_v2",
+  "dataset_revision": "cead6df28c96dabb53dd89714c779dff24bcdf49",
+  "engibench_version": "0.2.0+27055ae4248f",
   "metrics": [
     "mmd",
     "coverage",
@@ -25,15 +26,15 @@
     "reaches_reference_rate"
   ],
   "n_samples": 50,
-  "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volume_fraction_target budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.",
+  "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volfrac budget. v3: v2 on thermoelastic_2d_v2, where the volume target is named volfrac and the volume-fraction error is no longer an objective. Same rows, same draws, same metrics as v2.",
   "objective_weight_condition": "weight",
   "objective_weights": null,
   "problem_conditions": [
     "fixed_elements",
     "force_elements_x",
     "force_elements_y",
     "heatsink_elements",
-    "volume_fraction_target",
+    "volfrac",
     "rmin",
     "weight"
   ],
@@ -44,8 +45,7 @@
     3
   ],
   "sigma": null,
-  "version": "v2",
+  "version": "v3",
   "volfrac_tol": 0.01,
-  "volume_condition": "volume_fraction_target",
-  "aggregation": "mean"
+  "volume_condition": "volfrac"
 }

tests/test_checkpoint_packages.py

@@ -214,10 +214,10 @@ def fake_upload(**kwargs: Any) -> str:
         seed=1,
         checkpoint_files={},
         run_config={},
-        condition_keys=["volume_fraction_target", "rmin", "weight"],
+        condition_keys=["volfrac", "rmin", "weight"],
     )
 
-    assert uploads[0]["condition_keys"] == ["volume_fraction_target", "rmin", "weight"]
+    assert uploads[0]["condition_keys"] == ["volfrac", "rmin", "weight"]
 
 
 def test_recorded_schema_survives_the_problem_gaining_a_condition(tmp_path: Path) -> None:

tests/test_condition_schema.py

@@ -26,11 +26,11 @@
     "force_elements_x",
     "force_elements_y",
     "heatsink_elements",
-    "volume_fraction_target",
+    "volfrac",
     "rmin",
     "weight",
 )
-THERMOELASTIC_SCALARS = ("volume_fraction_target", "rmin", "weight")
+THERMOELASTIC_SCALARS = ("volfrac", "rmin", "weight")
 
 
 class _EchoGenerator(Generator):

tests/test_metrics_suite.py

@@ -261,12 +261,12 @@ def test_the_default_spec_metrics_are_all_registered() -> None:
 
 
 @pytest.mark.parametrize("problem_id", ["beams2d", "heatconduction2d", "photonics2d", "thermoelastic2d"])
-def test_every_v2_spec_names_only_registered_metrics(problem_id: str) -> None:
-    spec = EvalSpec.load(f"{problem_id}/v2")
-    assert spec.version == "v2"
+def test_every_current_spec_names_only_registered_metrics(problem_id: str) -> None:
+    spec = EvalSpec.load(problem_id)
     assert spec.aggregation == "mean"
     assert set(spec.metrics) <= set(METRICS)
 
 
-def test_the_default_spec_is_v2() -> None:
+def test_the_default_spec_is_the_newest_committed_for_the_problem() -> None:
     assert EvalSpec.load("beams2d").version == "v2"
+    assert EvalSpec.load("thermoelastic2d").version == "v3"

tests/test_specs.py

@@ -25,12 +25,13 @@
 SPEC_IDS = [f"{path.parent.name}/{path.stem}" for path in SPEC_PATHS]
 
 CURRENT_PATHS = [
-    max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: p.stem)
+    max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: int(p.stem[1:]))
     for problem in sorted({p.parent.name for p in SPEC_PATHS})
 ]
 CURRENT_IDS = [f"{path.parent.name}/{path.stem}" for path in CURRENT_PATHS]
-"""The newest spec per problem. Older versions stay committed so published rows can
-be read under the protocol that produced them, and may name metrics since retired."""
+"""The newest spec per problem. Older versions stay committed only while the pinned
+EngiBench still reproduces them. When EngiBench redefines a problem, its old spec
+versions are deleted and git history keeps them."""
 
 
 class _FakeConditions:
@@ -108,7 +109,7 @@ def test_digest_changes_when_a_condition_is_renamed() -> None:
     indices = np.array([0, 1, 2])
     designs = np.zeros((3, 4))
     before = _digest(indices, _FakeConditions({"volfrac": [0.3, 0.4, 0.5]}), designs)
-    after = _digest(indices, _FakeConditions({"volume": [0.3, 0.4, 0.5]}), designs)
+    after = _digest(indices, _FakeConditions({"target_volume": [0.3, 0.4, 0.5]}), designs)
     assert before != after
 
 
@@ -169,22 +170,21 @@ def test_a_differently_defined_problem_reports_itself() -> None:
     """
 
     class _Problem:
-        conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "weight"]
-        dataset_id = "IDEALLab/thermoelastic_2d_v0"
+        conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "forcedist"]
+        dataset_id = "IDEALLab/beams_2d_50_100_v1"
 
     spec = EvalSpec(
-        problem_id="thermoelastic2d",
-        problem_conditions=("volume_fraction_target", "rmin", "weight"),
-        dataset_id="IDEALLab/thermoelastic_2d_v1",
+        problem_id="beams2d",
+        problem_conditions=("volfrac", "rmin"),
+        dataset_id="IDEALLab/beams_2d_50_100_v0",
         engibench_version="0.2.0",
     )
     with pytest.raises(ProblemDefinitionMismatchError) as caught:
         spec.check_problem_definition(_Problem())
 
     message = str(caught.value)
-    assert "volume_fraction_target" in message
-    assert "volfrac" in message
-    assert "thermoelastic_2d_v0" in message
+    assert "forcedist" in message
+    assert "beams_2d_50_100_v1" in message
 
 
 def test_a_matching_problem_definition_passes() -> None: