diff --git a/README.md b/README.md index 4876c112f..04ccc5fca 100644 --- a/README.md +++ b/README.md @@ -602,6 +602,7 @@ database: # MAP-Elites quality-diversity population_size: 500 num_islands: 5 # Parallel evolution + # island_initial_programs: [null, "seeds/alt.py"] # Optional per-island starting programs migration_interval: 20 feature_dimensions: ["complexity", "diversity", "performance"] diff --git a/configs/README.md b/configs/README.md index 836c4b3ad..51755e913 100644 --- a/configs/README.md +++ b/configs/README.md @@ -37,6 +37,7 @@ database: num_islands: 5 # Number of separate populations migration_interval: 50 # Migrate every N generations migration_rate: 0.1 # Fraction of top programs to migrate + # island_initial_programs: [null, "seeds/greedy.py"] # Optional per-island starting programs ``` ### Parameter Guidelines @@ -44,6 +45,14 @@ database: - **num_islands**: 3-10 for most problems (more = more diversity) - **migration_interval**: 25-100 generations (higher = more independence) - **migration_rate**: 0.05-0.2 (5%-20%, higher = faster knowledge sharing) +- **island_initial_programs**: Start islands from different programs, e.g. distinct + algorithmic approaches. Entry *i* is the program island *i* starts from; `null` or a + missing entry keeps the default (the island starts empty and takes its first parent + from the other islands). Entry 0 must be `null`, because island 0 always starts from + the initial program given on the command line, and the list may not be longer than + `num_islands`. Relative paths resolve against the config file's directory. Each seed is + evaluated once at the start of a fresh run; resuming from a checkpoint does not add + them again. ### When to Use What diff --git a/configs/default_config.yaml b/configs/default_config.yaml index 6e1289f43..844624a50 100644 --- a/configs/default_config.yaml +++ b/configs/default_config.yaml @@ -103,6 +103,10 @@ database: population_size: 1000 # Maximum number of programs to keep in memory archive_size: 100 # Size of elite archive num_islands: 5 # Number of islands for island model (separate populations) + # Optional starting program per island index; null/missing starts the island as usual. + # Entry 0 must be null: island 0 always starts from the initial program given on the + # command line. Relative paths resolve against this config file's directory. + # island_initial_programs: [null, "seeds/greedy.py", null, "seeds/random_search.py"] # Island-based evolution parameters # Islands provide diversity by maintaining separate populations that evolve independently. diff --git a/configs/island_config_example.yaml b/configs/island_config_example.yaml index bd772a927..700a7d500 100644 --- a/configs/island_config_example.yaml +++ b/configs/island_config_example.yaml @@ -23,6 +23,10 @@ database: # Island-based evolution settings num_islands: 5 # Number of separate populations + # Optional starting program per island; null/missing starts the island as usual. + # Entry 0 must be null (island 0 starts from the command-line initial program). + # Relative paths resolve against this config file's directory. + # island_initial_programs: [null, "seeds/greedy.py", null, "seeds/random_search.py"] migration_interval: 50 # Migrate every 50 generations migration_rate: 0.1 # Migrate 10% of top programs diff --git a/openevolve/config.py b/openevolve/config.py index 2014802ff..560dfe058 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -335,6 +335,9 @@ class DatabaseConfig: population_size: int = 1000 archive_size: int = 100 num_islands: int = 5 + # Entry i is the program island i starts from; null or missing keeps the default. + # Island 0 always starts from the initial program passed on the command line. + island_initial_programs: List[Optional[str]] = field(default_factory=list) # Selection parameters elite_selection_ratio: float = 0.1 @@ -384,6 +387,19 @@ class DatabaseConfig: embedding_api_base: Optional[str] = None similarity_threshold: float = 0.99 + def validate_island_initial_programs(self) -> None: + """Check island_initial_programs against num_islands and the island 0 rule""" + if len(self.island_initial_programs) > self.num_islands: + raise ValueError( + "database.island_initial_programs has more entries than database.num_islands" + ) + + if self.island_initial_programs and self.island_initial_programs[0] is not None: + raise ValueError( + "database.island_initial_programs[0] must be null; island 0 starts from the " + "initial program passed on the command line" + ) + @dataclass class EvaluatorConfig: @@ -486,6 +502,16 @@ def from_yaml(cls, path: Union[str, Path]) -> "Config": if not template_path.is_absolute(): config.prompt.template_dir = str((config_path.parent / template_path).resolve()) + # Resolve island_initial_programs relative to config file location + config.database.island_initial_programs = [ + ( + str((config_path.parent / seed_path).resolve()) + if seed_path is not None and not Path(seed_path).is_absolute() + else seed_path + ) + for seed_path in config.database.island_initial_programs + ] + return config @classmethod @@ -531,6 +557,8 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": "prompt.island_system_messages has more entries than database.num_islands" ) + config.database.validate_island_initial_programs() + if config.prompt.programs_as_changes_description and not config.diff_based_evolution: raise ValueError( "prompt.programs_as_changes_description=true requires diff_based_evolution=true " diff --git a/openevolve/controller.py b/openevolve/controller.py index c0e5d386d..7977d05c0 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -119,6 +119,25 @@ def __init__( if not self.file_extension.startswith("."): self.file_extension = f".{self.file_extension}" + # Load per-island initial programs (island 0 always starts from the initial program). + # Validate again: a Config built in code never passes through Config.from_dict + self.config.database.validate_island_initial_programs() + self.island_initial_program_codes: Dict[int, str] = {} + for island_idx, seed_path in enumerate(self.config.database.island_initial_programs): + if seed_path is None: + continue + if not os.path.isfile(seed_path): + raise FileNotFoundError( + f"database.island_initial_programs[{island_idx}] '{seed_path}' does not exist" + ) + if Path(seed_path).suffix != Path(initial_program_path).suffix: + logger.warning( + f"Initial program for island {island_idx} '{seed_path}' has a different " + f"extension than '{initial_program_path}'" + ) + with open(seed_path, "r") as f: + self.island_initial_program_codes[island_idx] = f.read() + # Set the file_suffix in config (can be overridden in YAML) if not hasattr(self.config, "file_suffix") or self.config.file_suffix == ".py": self.config.file_suffix = self.file_extension @@ -250,6 +269,38 @@ def _load_initial_program(self) -> str: with open(self.initial_program_path, "r") as f: return f.read() + async def _add_initial_program( + self, code: str, iteration: int, target_island: Optional[int] = None + ) -> Program: + """Evaluate a starting program, add it to the database, and store its artifacts""" + program_id = str(uuid.uuid4()) + + # Evaluate the initial program + metrics = await self.evaluator.evaluate_program(code, program_id) + + program = Program( + id=program_id, + code=code, + changes_description=self.config.prompt.initial_changes_description, + language=self.config.language, + metrics=metrics, + iteration_found=iteration, + ) + + if self.database.add(program, target_island=target_island) is None: + logger.warning( + f"Population strategy rejected the initial program for island {target_island}" + ) + return program + + # Check for and store artifacts from initial program + artifacts = self.evaluator.get_pending_artifacts(program_id) + if artifacts: + self.database.store_artifacts(program_id, artifacts) + logger.info(f"Stored artifacts for initial program") + + return program + async def run( self, iterations: Optional[int] = None, @@ -291,29 +342,14 @@ async def run( if should_add_initial: logger.info("Adding initial program to database") - initial_program_id = str(uuid.uuid4()) - - # Evaluate the initial program - initial_metrics = await self.evaluator.evaluate_program( - self.initial_program_code, initial_program_id + initial_program = await self._add_initial_program( + self.initial_program_code, start_iteration ) + initial_metrics = initial_program.metrics - initial_program = Program( - id=initial_program_id, - code=self.initial_program_code, - changes_description=self.config.prompt.initial_changes_description, - language=self.config.language, - metrics=initial_metrics, - iteration_found=start_iteration, - ) - - self.database.add(initial_program) - - # Check for and store artifacts from initial program - initial_artifacts = self.evaluator.get_pending_artifacts(initial_program_id) - if initial_artifacts: - self.database.store_artifacts(initial_program_id, initial_artifacts) - logger.info(f"Stored artifacts for initial program") + for island_idx, seed_code in self.island_initial_program_codes.items(): + logger.info(f"Adding initial program for island {island_idx} to database") + await self._add_initial_program(seed_code, start_iteration, island_idx) # Check if combined_score is present in the metrics if "combined_score" not in initial_metrics: diff --git a/tests/test_island_initial_programs.py b/tests/test_island_initial_programs.py new file mode 100644 index 000000000..b7cea5e42 --- /dev/null +++ b/tests/test_island_initial_programs.py @@ -0,0 +1,274 @@ +"""Tests for seeding islands from per-island initial programs (issue #55).""" + +import asyncio +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +import yaml + +# Dummy API key so LLM ensembles initialize without a real key +os.environ.setdefault("OPENAI_API_KEY", "test") + +from openevolve.config import Config +from openevolve.controller import OpenEvolve +from openevolve.population import PopulationStrategy + + +class MockEvaluator: + """Mock evaluator that records the code it was asked to evaluate""" + + def __init__(self): + self.evaluated = [] + + async def evaluate_program(self, code, program_id): + self.evaluated.append(code) + return {"combined_score": 0.1 * len(self.evaluated)} + + def get_pending_artifacts(self, program_id): + return None + + +class TestIslandInitialProgramsConfig(unittest.TestCase): + def test_defaults_to_empty(self): + self.assertEqual(Config().database.island_initial_programs, []) + self.assertEqual(Config.from_dict({}).database.island_initial_programs, []) + + def test_accepts_null_entries(self): + config = Config.from_dict( + {"database": {"num_islands": 3, "island_initial_programs": [None, "b.py", None]}} + ) + self.assertEqual(config.database.island_initial_programs, [None, "b.py", None]) + + def test_rejects_more_entries_than_islands(self): + with self.assertRaisesRegex(ValueError, "island_initial_programs.*num_islands"): + Config.from_dict( + {"database": {"num_islands": 2, "island_initial_programs": [None, "b.py", "c.py"]}} + ) + + def test_rejects_non_null_entry_zero(self): + for entry in ("a.py", ""): + with self.subTest(entry=entry): + with self.assertRaisesRegex(ValueError, r"island_initial_programs\[0\]"): + Config.from_dict( + {"database": {"num_islands": 2, "island_initial_programs": [entry]}} + ) + + def test_relative_paths_resolve_against_config_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + config_dir = Path(tmpdir) / "config_subdir" + config_dir.mkdir() + config_file = config_dir / "config.yaml" + absolute_seed = str(Path(tmpdir) / "absolute.py") + with open(config_file, "w") as f: + yaml.dump( + { + "database": { + "num_islands": 4, + "island_initial_programs": [ + None, + "seeds/one.py", + absolute_seed, + None, + ], + } + }, + f, + ) + + config = Config.from_yaml(config_file) + + self.assertEqual( + config.database.island_initial_programs, + [None, str((config_dir / "seeds/one.py").resolve()), absolute_seed, None], + ) + + +class TestIslandInitialProgramsController(unittest.TestCase): + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.initial_program_path = self._write("initial_program.py", "def solve(): return 0\n") + self.seed_one_path = self._write("seed_one.py", "def solve(): return 1\n") + self.seed_three_path = self._write("seed_three.py", "def solve(): return 3\n") + self.evaluator_path = self._write( + "evaluator.py", "def evaluate(program_path):\n return {'combined_score': 0.5}\n" + ) + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _write(self, name, content): + path = os.path.join(self.test_dir, name) + with open(path, "w") as f: + f.write(content) + return path + + def _make_config(self, island_initial_programs=None, num_islands=4): + config = Config() + config.database.in_memory = True + config.database.num_islands = num_islands + if island_initial_programs is not None: + config.database.island_initial_programs = island_initial_programs + return config + + def _make_controller(self, config, **kwargs): + return OpenEvolve( + initial_program_path=self.initial_program_path, + evaluation_file=self.evaluator_path, + config=config, + output_dir=os.path.join(self.test_dir, "out"), + **kwargs, + ) + + def _run(self, config, checkpoint_path=None, **kwargs): + """Run the controller with mocked evaluation and no evolution iterations""" + + async def run(): + with ( + patch("openevolve.controller.Evaluator", return_value=MockEvaluator()), + patch("openevolve.controller.ProcessParallelController") as parallel_class, + ): + parallel = Mock() + parallel.run_evolution = AsyncMock(return_value=None) + parallel.shutdown_event.is_set.return_value = False + parallel.produced_no_candidates = False + parallel_class.return_value = parallel + + controller = self._make_controller(config, **kwargs) + await controller.run(iterations=0, checkpoint_path=checkpoint_path) + return controller + + return asyncio.run(run()) + + def _codes_by_island(self, database): + return [ + sorted(database.programs[pid].code for pid in island) for island in database.islands + ] + + def _read(self, path): + with open(path) as f: + return f.read() + + def test_seeds_land_on_their_islands(self): + config = self._make_config([None, self.seed_one_path, None, self.seed_three_path]) + controller = self._run(config) + database = controller.database + + self.assertEqual( + self._codes_by_island(database), + [ + [self._read(self.initial_program_path)], + [self._read(self.seed_one_path)], + [], + [self._read(self.seed_three_path)], + ], + ) + for island_idx, island in enumerate(database.islands): + for program_id in island: + self.assertEqual(database.programs[program_id].metadata["island"], island_idx) + self.assertEqual(len(controller.evaluator.evaluated), 3) + + def test_null_island_is_filled_lazily_as_before(self): + config = self._make_config([None, self.seed_one_path]) + database = self._run(config).database + self.assertEqual(database.islands[2], set()) + + database.current_island = 2 + parent = database._sample_exploration_parent() + + self.assertEqual(parent.code, database.get_best_program().code) + self.assertEqual(parent.metadata["island"], 2) + + def test_missing_seed_file_fails_at_construction(self): + missing = os.path.join(self.test_dir, "missing.py") + config = self._make_config([None, missing]) + with self.assertRaisesRegex(FileNotFoundError, r"island_initial_programs\[1\].*missing"): + self._make_controller(config) + + def test_config_built_in_code_is_validated_at_construction(self): + # Config objects built in code (e.g. for run_evolution) skip Config.from_dict + cases = { + "more entries": ([None, self.seed_one_path, self.seed_three_path], "num_islands"), + "entry zero": ([self.seed_one_path], r"island_initial_programs\[0\]"), + } + for name, (seeds, message) in cases.items(): + with self.subTest(name): + config = self._make_config(seeds, num_islands=2) + with self.assertRaisesRegex(ValueError, message): + self._make_controller(config) + + def test_different_extension_warns(self): + seed_path = self._write("seed.rs", "fn solve() {}\n") + config = self._make_config([None, seed_path]) + with self.assertLogs("openevolve.controller", level="WARNING") as logs: + controller = self._make_controller(config) + self.assertTrue(any("different extension" in line for line in logs.output)) + self.assertEqual(controller.island_initial_program_codes, {1: "fn solve() {}\n"}) + + def test_rejected_seed_warns_and_leaves_island_empty(self): + strategy = PopulationStrategy(admit=lambda snapshot, candidate, island: island == 0) + config = self._make_config([None, self.seed_one_path]) + with self.assertLogs("openevolve.controller", level="WARNING") as logs: + database = self._run(config, population_strategy=strategy).database + self.assertTrue(any("rejected the initial program" in line for line in logs.output)) + self.assertEqual(len(database.programs), 1) + self.assertEqual(database.islands[1], set()) + + def test_seed_artifacts_are_stored(self): + self.evaluator_path = self._write( + "evaluator.py", + "from openevolve.evaluation_result import EvaluationResult\n" + "\n" + "def evaluate(program_path):\n" + " with open(program_path) as f:\n" + " source = f.read()\n" + " return EvaluationResult(\n" + " metrics={'combined_score': 0.5}, artifacts={'source': source}\n" + " )\n", + ) + config = self._make_config([None, self.seed_one_path]) + controller = self._make_controller(config) + with patch.object( + controller, "_run_evolution_with_checkpoints", new=AsyncMock(return_value=None) + ): + asyncio.run(controller.run(iterations=0)) + + database = controller.database + (seed_id,) = database.islands[1] + self.assertEqual(database.get_artifacts(seed_id)["source"], self._read(self.seed_one_path)) + + def test_resume_does_not_duplicate_seeds(self): + config = self._make_config([None, self.seed_one_path, None, self.seed_three_path]) + controller = self._run(config) + controller._save_checkpoint(0) + checkpoint_path = os.path.join(controller.output_dir, "checkpoints", "checkpoint_0") + saved_codes = self._codes_by_island(controller.database) + + resumed_config = self._make_config([None, self.seed_one_path, None, self.seed_three_path]) + resumed = self._run(resumed_config, checkpoint_path=checkpoint_path) + + self.assertEqual(self._codes_by_island(resumed.database), saved_codes) + self.assertEqual(len(resumed.database.programs), 3) + self.assertEqual(resumed.evaluator.evaluated, []) + + def test_config_without_field_matches_previous_behaviour(self): + for island_initial_programs in (None, [], [None, None]): + with self.subTest(island_initial_programs=island_initial_programs): + controller = self._run(self._make_config(island_initial_programs)) + database = controller.database + + self.assertEqual( + self._codes_by_island(database), + [[self._read(self.initial_program_path)], [], [], []], + ) + (program,) = database.programs.values() + self.assertEqual(program.metadata["island"], 0) + self.assertEqual(database.best_program_id, program.id) + self.assertEqual(controller.evaluator.evaluated, [program.code]) + + +if __name__ == "__main__": + unittest.main()