diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 77a7b853..f7025b04 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -39,18 +39,19 @@ jobs: python-version: "3.12" - uses: actions/checkout@v4 - run: pip install -e ".[testing]" - # The committed eval specs are frozen against the current EngiBench, whose - # photonics2d and thermoelastic2d read the v1 datasets. The newest release - # on PyPI (0.2.0) still points those two at v0, so the PyPI build draws - # different conditions and no spec could reproduce itself. + # The committed eval specs are frozen against the pinned EngiBench, where every + # volume target is named volfrac: thermoelastic2d reads thermoelastic_2d_v2 and + # heatconduction2d reads heat_conduction_2d_v1. The newest release on PyPI + # (0.2.0) predates that, so the PyPI build draws different conditions and no + # spec could reproduce itself. # # --force-reinstall is required, not decorative: the git build reports the # same version string (0.2.0) as the PyPI one, so a plain install is a # no-op and the specs would be verified against the wrong EngiBench. # --no-deps keeps it from re-resolving the whole stack. - - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@0a028c03d02b266cb4394968c386e45781f4605b" + - run: pip install --force-reinstall --no-deps "engibench @ git+https://github.com/IDEALLab/EngiBench.git@27055ae4248f0594a71e63950c7336462e7decec" # Fail loudly if the pin silently stopped taking effect. - - run: python -c "import engibench.problems.thermoelastic2d.v0 as m; assert m.ThermoElastic2D.dataset_id.endswith('_v1'), m.ThermoElastic2D.dataset_id" + - run: python -c "import engibench.problems.thermoelastic2d.v0 as t, engibench.problems.heatconduction2d.v0 as h; assert t.ThermoElastic2D.dataset_id.endswith('_v2'), t.ThermoElastic2D.dataset_id; assert h.HeatConduction2D.dataset_id.endswith('_v1'), h.HeatConduction2D.dataset_id" # Every committed spec must still resolve: same problem definition, same # pinned dataset revision, same digest. - run: pytest tests/ -v diff --git a/CONTRIBUTING_A_MODEL.md b/CONTRIBUTING_A_MODEL.md index 0f0e78e5..6ace84eb 100644 --- a/CONTRIBUTING_A_MODEL.md +++ b/CONTRIBUTING_A_MODEL.md @@ -67,7 +67,7 @@ solver settings that are not dataset columns at all. Size your network from the ```python from engiopt.transforms import condition_keys -cond_keys = condition_keys(problem) # e.g. ("volume_fraction_target", "rmin", "weight") +cond_keys = condition_keys(problem) # e.g. ("volfrac", "rmin", "weight") n_conds = len(cond_keys) ``` diff --git a/engiopt/evaluate.py b/engiopt/evaluate.py index 8a54f97a..748a56e8 100644 --- a/engiopt/evaluate.py +++ b/engiopt/evaluate.py @@ -64,7 +64,7 @@ class Args: cgan_cnn_2d:023dd1fb gan_cnn_2d:06d9a9a1` asks for exactly the two that exist.""" spec: str | None = None - """Eval spec reference, e.g. `beams2d/v1`. Defaults to `/v1`.""" + """Eval spec reference, e.g. `beams2d/v2`. Defaults to the newest spec for the problem.""" metrics: tuple[str, ...] = () """Metric names; defaults to the spec's list.""" include_expensive: bool = False @@ -341,7 +341,7 @@ def main(args: Args) -> int: if args.list_generators or args.list_metrics: return 0 - spec = args.spec or f"{args.problem_id}/v1" + spec = args.spec or args.problem_id # no version: the newest spec for the problem evaluator = Evaluator.for_problem(args.problem_id, spec=spec) print(f"Problem {args.problem_id} | spec {evaluator.spec.version} | n={evaluator.spec.n_samples}") diff --git a/engiopt/evaluation/evaluator.py b/engiopt/evaluation/evaluator.py index 927f8090..24095139 100644 --- a/engiopt/evaluation/evaluator.py +++ b/engiopt/evaluation/evaluator.py @@ -128,7 +128,7 @@ def for_problem( Args: problem_id: EngiBench problem registry key. spec: `"/"`, an `EvalSpec`, or None to load - `"/v1"`. + the newest spec committed for the problem. device: Torch device; auto-selected when omitted. registry: Metric registry override, useful in tests. """ diff --git a/engiopt/evaluation/spec.py b/engiopt/evaluation/spec.py index 4af8e720..a39c017d 100644 --- a/engiopt/evaluation/spec.py +++ b/engiopt/evaluation/spec.py @@ -187,12 +187,15 @@ def __post_init__(self) -> None: @classmethod def load(cls, reference: str, *, root: Path | None = None) -> EvalSpec: - """Load a spec from `"/"` or a path to a JSON file.""" + """Load a spec from `"/"`, `""` for the newest version, or a JSON path.""" path = Path(reference) if path.suffix == ".json" and path.exists(): return cls(**json.loads(path.read_text())) problem_id, _, version = reference.partition("/") - version = version or "v2" + if not version: + # No version given: use the newest spec committed for this problem. + committed = sorted((root or SPEC_ROOT).glob(f"{problem_id}/v*.json"), key=lambda p: int(p.stem[1:])) + version = committed[-1].stem if committed else "v1" spec_path = (root or SPEC_ROOT) / problem_id / f"{version}.json" if not spec_path.exists(): raise FileNotFoundError( diff --git a/engiopt/specs/heatconduction2d/v1.json b/engiopt/specs/heatconduction2d/v1.json deleted file mode 100644 index a0221ecd..00000000 --- a/engiopt/specs/heatconduction2d/v1.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "condition_digest": "40030d1b2482e003", - "condition_seed": 1, - "dataset_id": "IDEALLab/heat_conduction_2d_v0", - "dataset_revision": "9f07c1e70d244f783e34a71c96db621173871dc1", - "engibench_version": "0.2.0+0a028c03d02b", - "metrics": [ - "mmd", - "dpp", - "novelty", - "cond_sens", - "viol", - "iog", - "cog", - "fog" - ], - "n_samples": 50, - "notes": "Feasibility = EngiBench constraints plus the volume budget.", - "objective_weight_condition": null, - "objective_weights": null, - "problem_conditions": [ - "volume", - "length" - ], - "problem_id": "heatconduction2d", - "required_seeds": [ - 1, - 2, - 3 - ], - "sigma": 10.0, - "version": "v1", - "volfrac_tol": 0.01, - "volume_condition": "volume" -} diff --git a/engiopt/specs/heatconduction2d/v2.json b/engiopt/specs/heatconduction2d/v3.json similarity index 56% rename from engiopt/specs/heatconduction2d/v2.json rename to engiopt/specs/heatconduction2d/v3.json index bdfed215..8fc45e9d 100644 --- a/engiopt/specs/heatconduction2d/v2.json +++ b/engiopt/specs/heatconduction2d/v3.json @@ -1,9 +1,10 @@ { - "condition_digest": "40030d1b2482e003", + "aggregation": "mean", + "condition_digest": "aa13508d396239d5", "condition_seed": 1, - "dataset_id": "IDEALLab/heat_conduction_2d_v0", - "dataset_revision": "9f07c1e70d244f783e34a71c96db621173871dc1", - "engibench_version": "0.2.0+0a028c03d02b", + "dataset_id": "IDEALLab/heat_conduction_2d_v1", + "dataset_revision": "9277ee7476626250e5c9f9ed12e0dd21d181b532", + "engibench_version": "0.2.0+27055ae4248f", "metrics": [ "mmd", "coverage", @@ -25,11 +26,11 @@ "reaches_reference_rate" ], "n_samples": 50, - "notes": "Feasibility = EngiBench constraints plus the volume budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.", + "notes": "Feasibility = EngiBench constraints plus the volume budget. v3: v2 on heat_conduction_2d_v1, where the volume target is named volfrac. Same rows, same draws, same metrics as v2.", "objective_weight_condition": null, "objective_weights": null, "problem_conditions": [ - "volume", + "volfrac", "length" ], "problem_id": "heatconduction2d", @@ -39,8 +40,7 @@ 3 ], "sigma": null, - "version": "v2", + "version": "v3", "volfrac_tol": 0.01, - "volume_condition": "volume", - "aggregation": "mean" + "volume_condition": "volfrac" } diff --git a/engiopt/specs/thermoelastic2d/v1.json b/engiopt/specs/thermoelastic2d/v1.json deleted file mode 100644 index 0271b3d6..00000000 --- a/engiopt/specs/thermoelastic2d/v1.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "condition_digest": "818d93c2a2d5ac25", - "condition_seed": 1, - "dataset_id": "IDEALLab/thermoelastic_2d_v1", - "dataset_revision": "7d5b4a694bd1a8f3f25fd745ac9f669764e29ea5", - "engibench_version": "0.2.0+0a028c03d02b", - "metrics": [ - "mmd", - "dpp", - "novelty", - "cond_sens", - "viol", - "iog", - "cog", - "fog" - ], - "n_samples": 50, - "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volume_fraction_target budget.", - "objective_weight_condition": "weight", - "objective_weights": null, - "problem_conditions": [ - "fixed_elements", - "force_elements_x", - "force_elements_y", - "heatsink_elements", - "volume_fraction_target", - "rmin", - "weight" - ], - "problem_id": "thermoelastic2d", - "required_seeds": [ - 1, - 2, - 3 - ], - "sigma": 10.0, - "version": "v1", - "volfrac_tol": 0.01, - "volume_condition": "volume_fraction_target" -} diff --git a/engiopt/specs/thermoelastic2d/v2.json b/engiopt/specs/thermoelastic2d/v3.json similarity index 55% rename from engiopt/specs/thermoelastic2d/v2.json rename to engiopt/specs/thermoelastic2d/v3.json index 977cbd97..a9c1bab8 100644 --- a/engiopt/specs/thermoelastic2d/v2.json +++ b/engiopt/specs/thermoelastic2d/v3.json @@ -1,9 +1,10 @@ { - "condition_digest": "818d93c2a2d5ac25", + "aggregation": "mean", + "condition_digest": "3665bcd7e957e3e6", "condition_seed": 1, - "dataset_id": "IDEALLab/thermoelastic_2d_v1", - "dataset_revision": "7d5b4a694bd1a8f3f25fd745ac9f669764e29ea5", - "engibench_version": "0.2.0+0a028c03d02b", + "dataset_id": "IDEALLab/thermoelastic_2d_v2", + "dataset_revision": "cead6df28c96dabb53dd89714c779dff24bcdf49", + "engibench_version": "0.2.0+27055ae4248f", "metrics": [ "mmd", "coverage", @@ -25,7 +26,7 @@ "reaches_reference_rate" ], "n_samples": 50, - "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volume_fraction_target budget. v2: the full metric suite on the one-decorator contract; `dpp` is now the n-th-root (geometric-mean) form and is not comparable with v1's raw determinant; `novelty` became `train_distance`; aggregation is declared here and recorded in every row.", + "notes": "Objectives combined per-sample by the `weight` condition. Feasibility = EngiBench constraints plus the volfrac budget. v3: v2 on thermoelastic_2d_v2, where the volume target is named volfrac and the volume-fraction error is no longer an objective. Same rows, same draws, same metrics as v2.", "objective_weight_condition": "weight", "objective_weights": null, "problem_conditions": [ @@ -33,7 +34,7 @@ "force_elements_x", "force_elements_y", "heatsink_elements", - "volume_fraction_target", + "volfrac", "rmin", "weight" ], @@ -44,8 +45,7 @@ 3 ], "sigma": null, - "version": "v2", + "version": "v3", "volfrac_tol": 0.01, - "volume_condition": "volume_fraction_target", - "aggregation": "mean" + "volume_condition": "volfrac" } diff --git a/tests/test_checkpoint_packages.py b/tests/test_checkpoint_packages.py index c55ba65d..f53b468e 100644 --- a/tests/test_checkpoint_packages.py +++ b/tests/test_checkpoint_packages.py @@ -214,10 +214,10 @@ def fake_upload(**kwargs: Any) -> str: seed=1, checkpoint_files={}, run_config={}, - condition_keys=["volume_fraction_target", "rmin", "weight"], + condition_keys=["volfrac", "rmin", "weight"], ) - assert uploads[0]["condition_keys"] == ["volume_fraction_target", "rmin", "weight"] + assert uploads[0]["condition_keys"] == ["volfrac", "rmin", "weight"] def test_recorded_schema_survives_the_problem_gaining_a_condition(tmp_path: Path) -> None: diff --git a/tests/test_condition_schema.py b/tests/test_condition_schema.py index f18ac209..8f667d68 100644 --- a/tests/test_condition_schema.py +++ b/tests/test_condition_schema.py @@ -26,11 +26,11 @@ "force_elements_x", "force_elements_y", "heatsink_elements", - "volume_fraction_target", + "volfrac", "rmin", "weight", ) -THERMOELASTIC_SCALARS = ("volume_fraction_target", "rmin", "weight") +THERMOELASTIC_SCALARS = ("volfrac", "rmin", "weight") class _EchoGenerator(Generator): diff --git a/tests/test_metrics_suite.py b/tests/test_metrics_suite.py index 603b03f7..95336185 100644 --- a/tests/test_metrics_suite.py +++ b/tests/test_metrics_suite.py @@ -261,12 +261,12 @@ def test_the_default_spec_metrics_are_all_registered() -> None: @pytest.mark.parametrize("problem_id", ["beams2d", "heatconduction2d", "photonics2d", "thermoelastic2d"]) -def test_every_v2_spec_names_only_registered_metrics(problem_id: str) -> None: - spec = EvalSpec.load(f"{problem_id}/v2") - assert spec.version == "v2" +def test_every_current_spec_names_only_registered_metrics(problem_id: str) -> None: + spec = EvalSpec.load(problem_id) assert spec.aggregation == "mean" assert set(spec.metrics) <= set(METRICS) -def test_the_default_spec_is_v2() -> None: +def test_the_default_spec_is_the_newest_committed_for_the_problem() -> None: assert EvalSpec.load("beams2d").version == "v2" + assert EvalSpec.load("thermoelastic2d").version == "v3" diff --git a/tests/test_specs.py b/tests/test_specs.py index b2bf4974..d5a1c94e 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -25,12 +25,13 @@ SPEC_IDS = [f"{path.parent.name}/{path.stem}" for path in SPEC_PATHS] CURRENT_PATHS = [ - max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: p.stem) + max(SPEC_ROOT.glob(f"{problem}/*.json"), key=lambda p: int(p.stem[1:])) for problem in sorted({p.parent.name for p in SPEC_PATHS}) ] CURRENT_IDS = [f"{path.parent.name}/{path.stem}" for path in CURRENT_PATHS] -"""The newest spec per problem. Older versions stay committed so published rows can -be read under the protocol that produced them, and may name metrics since retired.""" +"""The newest spec per problem. Older versions stay committed only while the pinned +EngiBench still reproduces them. When EngiBench redefines a problem, its old spec +versions are deleted and git history keeps them.""" class _FakeConditions: @@ -108,7 +109,7 @@ def test_digest_changes_when_a_condition_is_renamed() -> None: indices = np.array([0, 1, 2]) designs = np.zeros((3, 4)) before = _digest(indices, _FakeConditions({"volfrac": [0.3, 0.4, 0.5]}), designs) - after = _digest(indices, _FakeConditions({"volume": [0.3, 0.4, 0.5]}), designs) + after = _digest(indices, _FakeConditions({"target_volume": [0.3, 0.4, 0.5]}), designs) assert before != after @@ -169,22 +170,21 @@ def test_a_differently_defined_problem_reports_itself() -> None: """ class _Problem: - conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "weight"] - dataset_id = "IDEALLab/thermoelastic_2d_v0" + conditions_keys: ClassVar[list[str]] = ["volfrac", "rmin", "forcedist"] + dataset_id = "IDEALLab/beams_2d_50_100_v1" spec = EvalSpec( - problem_id="thermoelastic2d", - problem_conditions=("volume_fraction_target", "rmin", "weight"), - dataset_id="IDEALLab/thermoelastic_2d_v1", + problem_id="beams2d", + problem_conditions=("volfrac", "rmin"), + dataset_id="IDEALLab/beams_2d_50_100_v0", engibench_version="0.2.0", ) with pytest.raises(ProblemDefinitionMismatchError) as caught: spec.check_problem_definition(_Problem()) message = str(caught.value) - assert "volume_fraction_target" in message - assert "volfrac" in message - assert "thermoelastic_2d_v0" in message + assert "forcedist" in message + assert "beams_2d_50_100_v1" in message def test_a_matching_problem_definition_passes() -> None: