diff --git a/README.md b/README.md index d0d1bcc3c3..785a1d5804 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,8 @@ *Turn your LLMs into autonomous code optimizers that discover breakthrough algorithms*

- GitHub stars PyPI version - PyPI downloads + PyPI Downloads License

diff --git a/openevolve/_version.py b/openevolve/_version.py index 179eae7e3b..d7586dad44 100644 --- a/openevolve/_version.py +++ b/openevolve/_version.py @@ -1,3 +1,3 @@ """Version information for openevolve package.""" -__version__ = "0.3.1" +__version__ = "0.3.2" diff --git a/openevolve/controller.py b/openevolve/controller.py index 2a551cdaaa..a3f096bf8b 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -25,34 +25,6 @@ logger = logging.getLogger(__name__) -def _format_metrics(metrics: Dict[str, Any]) -> str: - """Safely format metrics, handling both numeric and string values""" - formatted_parts = [] - for name, value in metrics.items(): - if isinstance(value, (int, float)) and not isinstance(value, bool): - try: - formatted_parts.append(f"{name}={value:.4f}") - except (ValueError, TypeError): - formatted_parts.append(f"{name}={value}") - else: - formatted_parts.append(f"{name}={value}") - return ", ".join(formatted_parts) - - -def _format_improvement(improvement: Dict[str, Any]) -> str: - """Safely format improvement metrics""" - formatted_parts = [] - for name, diff in improvement.items(): - if isinstance(diff, (int, float)) and not isinstance(diff, bool): - try: - formatted_parts.append(f"{name}={diff:+.4f}") - except (ValueError, TypeError): - formatted_parts.append(f"{name}={diff}") - else: - formatted_parts.append(f"{name}={diff}") - return ", ".join(formatted_parts) - - class OpenEvolve: """ Main controller for OpenEvolve diff --git a/openevolve/utils/format_utils.py b/openevolve/utils/format_utils.py index 4dd83c7ac0..66926423fd 100644 --- a/openevolve/utils/format_utils.py +++ b/openevolve/utils/format_utils.py @@ -20,8 +20,10 @@ def format_metrics_safe(metrics: Dict[str, Any]) -> str: formatted_parts = [] for name, value in metrics.items(): - # Check if value is numeric (int, float) - if isinstance(value, (int, float)): + # bool is a subclass of int, but logging it as 1.0000/0.0000 is confusing. + if isinstance(value, bool): + formatted_parts.append(f"{name}={value}") + elif isinstance(value, (int, float)): try: # Only apply float formatting to numeric values formatted_parts.append(f"{name}={value:.4f}") @@ -54,7 +56,12 @@ def format_improvement_safe(parent_metrics: Dict[str, Any], child_metrics: Dict[ if metric in parent_metrics: parent_value = parent_metrics[metric] # Only calculate improvement for numeric values - if isinstance(child_value, (int, float)) and isinstance(parent_value, (int, float)): + if ( + isinstance(child_value, (int, float)) + and isinstance(parent_value, (int, float)) + and not isinstance(child_value, bool) + and not isinstance(parent_value, bool) + ): try: diff = child_value - parent_value improvement_parts.append(f"{metric}={diff:+.4f}") diff --git a/openevolve/utils/metrics_utils.py b/openevolve/utils/metrics_utils.py index 3efd18e25b..5c176e0e2e 100644 --- a/openevolve/utils/metrics_utils.py +++ b/openevolve/utils/metrics_utils.py @@ -21,7 +21,10 @@ def safe_numeric_average(metrics: Dict[str, Any]) -> float: numeric_values = [] for value in metrics.values(): - if isinstance(value, (int, float)): + # bool is a subclass of int, but a flag is not a score. Averaging it in + # would let e.g. {"error": 0.0, "timeout": True} score 0.5 instead of 0.0, + # giving a program that timed out a mid-range fitness. + if isinstance(value, (int, float)) and not isinstance(value, bool): try: # Convert to float and check if it's a valid number float_val = float(value) @@ -99,7 +102,8 @@ def get_fitness_score( for key, value in metrics.items(): # Exclude MAP feature dimensions from fitness calculation if key not in feature_dimensions: - if isinstance(value, (int, float)): + # Booleans are flags, not scores - see the note in safe_numeric_average. + if isinstance(value, (int, float)) and not isinstance(value, bool): try: float_val = float(value) if not (float_val != float_val): # Check for NaN diff --git a/tests/test_boolean_metrics.py b/tests/test_boolean_metrics.py new file mode 100644 index 0000000000..02a0d7ac56 --- /dev/null +++ b/tests/test_boolean_metrics.py @@ -0,0 +1,75 @@ +""" +Booleans in a metrics dict are FLAGS, not scores. + +`bool` is a subclass of `int`, so a naive `isinstance(value, (int, float))` check +silently treats True/False as 1.0/0.0. That matters in two places: + + * display - `timeout=1.0000` instead of `timeout=True` + * FITNESS - a program that timed out returns {"error": 0.0, "timeout": True} + (openevolve/evaluator.py), which averaged to 0.5 instead of 0.0, + handing a failed program a mid-range score. + +OpenEvolve's own evaluator emits `timeout: True` in several places, so this is not +a hypothetical input. +""" + +import os +import unittest + +os.environ.setdefault("OPENAI_API_KEY", "test") + +from openevolve.utils.format_utils import format_improvement_safe, format_metrics_safe +from openevolve.utils.metrics_utils import get_fitness_score, safe_numeric_average + + +class TestBooleanMetricsFormatting(unittest.TestCase): + def test_bool_rendered_as_true_false(self): + self.assertEqual( + format_metrics_safe({"valid": True, "timeout": False, "score": 0.25}), + "valid=True, timeout=False, score=0.2500", + ) + + def test_bool_excluded_from_improvement(self): + """A boolean flipping False->True is not a '+1.0000' improvement.""" + self.assertEqual( + format_improvement_safe( + {"valid": False, "score": 0.25}, + {"valid": True, "score": 0.5}, + ), + "score=+0.2500", + ) + + def test_numeric_formatting_unchanged(self): + self.assertEqual(format_metrics_safe({"score": 0.5, "n": 3}), "score=0.5000, n=3.0000") + + +class TestBooleanMetricsExcludedFromFitness(unittest.TestCase): + def test_timed_out_program_scores_zero(self): + """The exact dict openevolve/evaluator.py returns on timeout.""" + metrics = {"error": 0.0, "timeout": True} + # Before the fix both of these returned 0.5. + self.assertEqual(safe_numeric_average(metrics), 0.0) + self.assertEqual(get_fitness_score(metrics), 0.0) + + def test_bool_does_not_inflate_average(self): + # Without the guard this would be (0.4 + 1.0) / 2 = 0.7 + self.assertAlmostEqual(safe_numeric_average({"score": 0.4, "valid": True}), 0.4) + + def test_all_boolean_metrics_average_to_zero(self): + self.assertEqual(safe_numeric_average({"valid": True, "timeout": False}), 0.0) + + def test_combined_score_still_takes_precedence(self): + self.assertAlmostEqual(get_fitness_score({"combined_score": 0.9, "timeout": True}), 0.9) + + def test_ordinary_metrics_unaffected(self): + self.assertAlmostEqual(safe_numeric_average({"a": 0.8, "b": 0.6}), 0.7) + self.assertAlmostEqual(get_fitness_score({"a": 0.8, "b": 0.6}), 0.7) + + def test_feature_dimensions_still_excluded(self): + """Bool handling must not disturb the existing feature-dimension exclusion.""" + metrics = {"score": 0.8, "complexity": 100.0} + self.assertAlmostEqual(get_fitness_score(metrics, ["complexity"]), 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_format_utils.py b/tests/test_format_utils.py new file mode 100644 index 0000000000..d9d8751f25 --- /dev/null +++ b/tests/test_format_utils.py @@ -0,0 +1,15 @@ +from openevolve.utils.format_utils import format_improvement_safe, format_metrics_safe + + +def test_format_metrics_safe_preserves_boolean_values(): + assert format_metrics_safe({"valid": True, "score": 0.25}) == "valid=True, score=0.2500" + + +def test_format_improvement_safe_ignores_boolean_diffs(): + assert ( + format_improvement_safe( + {"valid": False, "score": 0.25}, + {"valid": True, "score": 0.5}, + ) + == "score=+0.2500" + )