Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion openevolve/prompts/defaults/fragments.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"attempt_all_metrics_improved": "Improvement in all metrics",
"attempt_all_metrics_regressed": "Regression in all metrics",
"attempt_mixed_metrics": "Mixed results",
"top_program_metrics_prefix": "Performs well on",
"top_program_metrics_prefix": "Metric:",
"diverse_program_metrics_prefix": "Alternative approach to",
"inspiration_type_diverse": "Diverse",
"inspiration_type_migrant": "Migrant",
Expand Down
54 changes: 54 additions & 0 deletions tests/test_prompt_sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@
Tests for PromptSampler in openevolve.prompt.sampler
"""

import json
import tempfile
import unittest
from pathlib import Path

from openevolve.config import Config
from openevolve.prompt.sampler import PromptSampler

Expand Down Expand Up @@ -49,6 +53,56 @@ def test_build_prompt(self):
# Check that the score value appears in the prompt (either as fitness or metrics)
self.assertIn("0.5", prompt["user"])

def test_top_program_metrics_use_neutral_descriptions(self):
"""Report metric values without inferring that they indicate good performance."""
cases = [
("score", 0.0, "0.0000"),
("score", -1.0, "-1.0000"),
("error", "evaluation failed", "evaluation failed"),
("latency", 10000.0, "10000.0000"),
]
for name, value, formatted_value in cases:
with self.subTest(metric=name, value=value):
prompt = self.prompt_sampler.build_prompt(
top_programs=[{"id": "top1", "metrics": {name: value}}],
)

self.assertIn(f"Key features: Metric: {name} ({formatted_value})", prompt["user"])
self.assertNotIn("Performs well on", prompt["user"])

def test_top_program_explicit_key_features_are_preserved(self):
"""Keep caller-provided descriptions instead of generating metric descriptions."""
prompt = self.prompt_sampler.build_prompt(
top_programs=[
{
"id": "top1",
"metrics": {"score": 0.0},
"key_features": ["Uses memoization", "Avoids repeated work"],
}
],
)

self.assertIn("Key features: Uses memoization, Avoids repeated work", prompt["user"])
self.assertNotIn("Metric: score", prompt["user"])

def test_top_program_metrics_prefix_can_be_overridden(self):
"""Load a custom metric prefix through the configured template directory."""
with tempfile.TemporaryDirectory() as template_dir:
fragments_path = Path(template_dir) / "fragments.json"
fragments_path.write_text(
json.dumps({"top_program_metrics_prefix": "Observed:"}), encoding="utf-8"
)
config = Config()
config.prompt.template_dir = template_dir
sampler = PromptSampler(config.prompt)

prompt = sampler.build_prompt(
top_programs=[{"id": "top1", "metrics": {"score": 0.0}}],
)

self.assertIn("Key features: Observed: score (0.0000)", prompt["user"])
self.assertNotIn("Metric: score", prompt["user"])


if __name__ == "__main__":
unittest.main()
Loading