From a4bbbd4d03b8fd89a29456107f6b970020c05860 Mon Sep 17 00:00:00 2001 From: keeponlight Date: Fri, 23 Oct 2026 23:28:36 +0800 Subject: [PATCH] fix(prompt): use a neutral label for top-program metrics --- openevolve/prompts/defaults/fragments.json | 2 +- tests/test_prompt_sampler.py | 54 ++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/openevolve/prompts/defaults/fragments.json b/openevolve/prompts/defaults/fragments.json index b59410405a..0ecfba065b 100644 --- a/openevolve/prompts/defaults/fragments.json +++ b/openevolve/prompts/defaults/fragments.json @@ -22,7 +22,7 @@ "attempt_all_metrics_improved": "Improvement in all metrics", "attempt_all_metrics_regressed": "Regression in all metrics", "attempt_mixed_metrics": "Mixed results", - "top_program_metrics_prefix": "Performs well on", + "top_program_metrics_prefix": "Metric:", "diverse_program_metrics_prefix": "Alternative approach to", "inspiration_type_diverse": "Diverse", "inspiration_type_migrant": "Migrant", diff --git a/tests/test_prompt_sampler.py b/tests/test_prompt_sampler.py index fc29e97306..03374617e8 100644 --- a/tests/test_prompt_sampler.py +++ b/tests/test_prompt_sampler.py @@ -2,7 +2,11 @@ Tests for PromptSampler in openevolve.prompt.sampler """ +import json +import tempfile import unittest +from pathlib import Path + from openevolve.config import Config from openevolve.prompt.sampler import PromptSampler @@ -49,6 +53,56 @@ def test_build_prompt(self): # Check that the score value appears in the prompt (either as fitness or metrics) self.assertIn("0.5", prompt["user"]) + def test_top_program_metrics_use_neutral_descriptions(self): + """Report metric values without inferring that they indicate good performance.""" + cases = [ + ("score", 0.0, "0.0000"), + ("score", -1.0, "-1.0000"), + ("error", "evaluation failed", "evaluation failed"), + ("latency", 10000.0, "10000.0000"), + ] + for name, value, formatted_value in cases: + with self.subTest(metric=name, value=value): + prompt = self.prompt_sampler.build_prompt( + top_programs=[{"id": "top1", "metrics": {name: value}}], + ) + + self.assertIn(f"Key features: Metric: {name} ({formatted_value})", prompt["user"]) + self.assertNotIn("Performs well on", prompt["user"]) + + def test_top_program_explicit_key_features_are_preserved(self): + """Keep caller-provided descriptions instead of generating metric descriptions.""" + prompt = self.prompt_sampler.build_prompt( + top_programs=[ + { + "id": "top1", + "metrics": {"score": 0.0}, + "key_features": ["Uses memoization", "Avoids repeated work"], + } + ], + ) + + self.assertIn("Key features: Uses memoization, Avoids repeated work", prompt["user"]) + self.assertNotIn("Metric: score", prompt["user"]) + + def test_top_program_metrics_prefix_can_be_overridden(self): + """Load a custom metric prefix through the configured template directory.""" + with tempfile.TemporaryDirectory() as template_dir: + fragments_path = Path(template_dir) / "fragments.json" + fragments_path.write_text( + json.dumps({"top_program_metrics_prefix": "Observed:"}), encoding="utf-8" + ) + config = Config() + config.prompt.template_dir = template_dir + sampler = PromptSampler(config.prompt) + + prompt = sampler.build_prompt( + top_programs=[{"id": "top1", "metrics": {"score": 0.0}}], + ) + + self.assertIn("Key features: Observed: score (0.0000)", prompt["user"]) + self.assertNotIn("Metric: score", prompt["user"]) + if __name__ == "__main__": unittest.main()