diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index bef4c8423..4f757cf3e 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -3,8 +3,9 @@ This is the evaluator-contract slice of [#468](https://github.com/algorithmicsuperintelligence/openevolve/issues/468). It runs the **unchanged** `program.py` and `evaluate.py` from Google's [AlphaEvolve examples](https://github.com/Google-Cloud-AI/alphaevolve-on-googlecloud) -through an explicit adapter. It does not replace the cloud SDK, translate cloud -configuration, export bundles, or implement the rest of #468. +through an explicit adapter. A separate opt-in helper imports a limited subset +of an experiment configuration for local Python runs. Neither helper replaces +the cloud SDK, exports bundles, or implements all of #468. ## Evaluate an upstream seed locally @@ -82,6 +83,65 @@ The adapter provides **no sandbox**. Like the original local evaluators, it runs trusted Python code with the current process's privileges. Use a separate execution environment for untrusted candidates. +## Import a local Python experiment configuration + +`config_from_alphaevolve` imports an explicit experiment dictionary without +reading `.env`, contacting Google, or creating model clients: + +```python +from openevolve.compat.alphaevolve import config_from_alphaevolve + +experiment = { + "title": "Circle Packing", + "program_language": "python", + "problem_description": "Pack circles into a unit square.", + "run_settings": {"max_programs": 10}, + "generation_settings": { + "models": [{"name": "my-local-model", "weight": 1}], + }, +} +config = config_from_alphaevolve( + experiment, + api_base="http://localhost:8000/v1", + api_key="local-placeholder", # Replace if the chosen endpoint requires authentication. +) +# Pass config to run_evolution(program_path, adapted_evaluator, config=config). +# Running evolution invokes your chosen endpoint; conversion alone does not. +``` + +The imported names must be served by the explicitly supplied OpenAI-compatible +endpoint. Gemini model names are not remapped to local models. Endpoint/key +values embedded in the cloud dictionary are never copied. A missing `api_key` +retains OpenEvolve's usual provider behavior, which may use `OPENAI_API_KEY` when +the client is subsequently constructed. Conversion adds no Google dependency. +For a Google SDK config object, pass its `model_dump()` dictionary. + +| AlphaEvolve field | OpenEvolve mapping | +| --- | --- | +| `generation_settings.models[].name/weight` | `llm.models`, with relative weights unchanged; null/missing weights become 1 | +| `run_settings.max_programs` | `max_iterations`, requiring an explicit positive integer | +| `problem_description` + optional `generation_settings.context` | Literal `prompt.system_message` text | +| `program_language="python"` | `language="python"`, `.py` suffix | + +The converter disables cascade evaluation for the single-stage local evaluator; +other OE settings retain their defaults. It does not mutate the input or start +a run. Missing models, missing budgets and unsupported languages raise instead +of guessing defaults. `title` is metadata and is not imported. + +`max_iterations` counts generation attempts, **not** successful evaluations, +requests, tokens or money. OE also evaluates the initial seed, and retries can +make additional calls. `run_settings.max_duration` is rejected because this +helper cannot enforce that wall-clock limit. Non-null unsupported settings emit +warnings naming the fields, including cloud `concurrency`, prompt-inclusion +flags and evolution settings. Their semantics are not silently presented as +equivalent OE controls; OE's worker count stays at its default of one unless +you configure it separately. Model-specific extra fields also warn and are not +forwarded. Warning messages include field paths, not their values. + +This is a subset of inbound configuration support (A3 in #468), not `.env` +translation, an experiment/controller shim, or a cloud-compatible export. +Use the returned `Config` directly with the Python API. + ## Validation Default unit tests use synthetic envelopes and OpenEvolve's real evaluator and @@ -89,6 +149,7 @@ callable-serialization path, without Google's SDK, keys, or network access: ```bash python -m unittest discover -s tests -p test_alphaevolve_compat.py +python -m unittest discover -s tests -p test_alphaevolve_config.py ``` The walkthrough additionally permits direct validation against an upstream diff --git a/openevolve/compat/alphaevolve.py b/openevolve/compat/alphaevolve.py index d9beef688..8dbb03807 100644 --- a/openevolve/compat/alphaevolve.py +++ b/openevolve/compat/alphaevolve.py @@ -2,12 +2,14 @@ import json import math +import warnings from collections.abc import Mapping from dataclasses import dataclass from numbers import Real from pathlib import Path from typing import Any, Callable, Optional +from openevolve.config import Config from openevolve.evaluation_result import EvaluationResult @@ -126,3 +128,104 @@ def adapt_evaluator( return _AlphaEvolveEvaluator( evaluator, objective_metric, failure_sentinel, float(failure_score) ) + + +def _config_mapping(value: Any, path: str) -> Mapping: + if not isinstance(value, Mapping): + raise ValueError(f"{path} must be a mapping") + return value + + +def _warn_unmapped_fields(value: Mapping, supported: set, prefix: str = "") -> None: + paths = [f"{prefix}{key}" for key in value if key not in supported and value[key] is not None] + if paths: + warnings.warn( + "AlphaEvolve settings not imported: " + ", ".join(paths), + UserWarning, + stacklevel=3, + ) + + +def config_from_alphaevolve( + experiment: Mapping, *, api_base: str, api_key: Optional[str] = None +) -> Config: + """Import an experiment dictionary for local, single-file Python evaluation. + + Requires explicit models and a positive run_settings.max_programs; it never + guesses a model or falls back to OE's 10,000-iteration default. Model names + and relative weights are preserved. The endpoint and optional key must come + from the caller, not Google Cloud fields. No files, clients or requests are + created. Pass SDK models as model_dump() dictionaries. + + max_programs becomes max_iterations (generation attempts, plus OE's seed + evaluation), not a strict evaluation, request, token, or cost limit. Optional + settings without mappings warn; max_duration is rejected rather than silently + losing a wall-clock limit. Other OE settings retain their defaults, except + cascade evaluation is disabled for the single-stage local evaluator adapter. + """ + experiment = _config_mapping(experiment, "experiment") + run = _config_mapping(experiment.get("run_settings"), "run_settings") + generation = _config_mapping(experiment.get("generation_settings"), "generation_settings") + if not isinstance(api_base, str) or not api_base.strip(): + raise ValueError("api_base must explicitly name the OpenEvolve model endpoint") + if experiment.get("program_language") != "python": + raise ValueError("Only program_language='python' is supported by this local import") + description = experiment.get("problem_description") + if not isinstance(description, str) or not description.strip(): + raise ValueError("problem_description must be a non-empty string") + iterations = run.get("max_programs") + if isinstance(iterations, bool) or not isinstance(iterations, int) or iterations < 1: + raise ValueError("run_settings.max_programs must be an explicit positive integer") + if run.get("max_duration") is not None: + raise ValueError("run_settings.max_duration cannot be enforced by this import") + + mixture = generation.get("models") + if not isinstance(mixture, (list, tuple)) or not mixture: + raise ValueError("generation_settings.models must be a non-empty list") + models = [] + for index, entry in enumerate(mixture): + path = f"generation_settings.models[{index}]" + entry = _config_mapping(entry, path) + name = entry.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"{path}.name must be a non-empty string") + weight = entry.get("weight") + if weight is None: + weight = 1.0 + if ( + isinstance(weight, bool) + or not isinstance(weight, Real) + or not math.isfinite(weight) + or weight <= 0 + ): + raise ValueError(f"{path}.weight must be finite and positive") + models.append({"name": name, "weight": float(weight)}) + _warn_unmapped_fields(entry, {"name", "weight"}, path + ".") + if not math.isfinite(sum(model["weight"] for model in models)): + raise ValueError("generation_settings.models weights must have a finite sum") + + # Prefix the text so a description like 'system_message' remains literal, + # rather than accidentally selecting a PromptSampler template of that name. + system_message = "Problem description:\n" + description + context = generation.get("context") + if context is not None: + if not isinstance(context, str): + raise ValueError("generation_settings.context must be a string or None") + if context: + system_message += "\n\nAdditional context:\n" + context + + _warn_unmapped_fields( + experiment, + {"title", "problem_description", "program_language", "run_settings", "generation_settings"}, + ) + _warn_unmapped_fields(run, {"max_programs", "max_duration"}, "run_settings.") + _warn_unmapped_fields(generation, {"models", "context"}, "generation_settings.") + return Config.from_dict( + { + "max_iterations": iterations, + "language": "python", + "llm": {"api_base": api_base, "api_key": api_key, "models": models}, + "prompt": {"system_message": system_message}, + "evaluator": {"cascade_evaluation": False}, + } + ) diff --git a/tests/test_alphaevolve_config.py b/tests/test_alphaevolve_config.py new file mode 100644 index 000000000..f12d50700 --- /dev/null +++ b/tests/test_alphaevolve_config.py @@ -0,0 +1,212 @@ +"""Offline configuration import for local Python AlphaEvolve examples (#468).""" + +import copy +import tempfile +import unittest +import warnings + +from openevolve.api import run_evolution +from openevolve.compat.alphaevolve import adapt_evaluator, config_from_alphaevolve +from openevolve.config import Config +from openevolve.prompt.sampler import PromptSampler + + +def experiment(): + return { + "title": "Circle Packing", + "program_language": "python", + "problem_description": "Pack circles into a unit square.", + "run_settings": {"max_programs": 10}, + "generation_settings": { + "models": [{"name": "local-fast", "weight": 7}, {"name": "local-large", "weight": 3}] + }, + } + + +class OfflineModel: + def __init__(self, config): + self.model = config.name + + async def generate_with_context(self, system_message, messages, **kwargs): + assert "Pack circles into a unit square." in system_message + return "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE" + + +def offline_model(config): + return OfflineModel(config) + + +class TestAlphaEvolveConfig(unittest.TestCase): + def convert(self, data, **kwargs): + return config_from_alphaevolve(data, api_base="http://localhost:8080/v1", **kwargs) + + def test_imports_models_budget_language_and_problem_without_mutating_input(self): + data = experiment() + original = copy.deepcopy(data) + config = self.convert(data, api_key="local-placeholder") + self.assertEqual(data, original) + self.assertEqual(config.max_iterations, 10) + self.assertEqual(config.language, "python") + self.assertEqual(config.file_suffix, ".py") + self.assertEqual( + [(m.name, m.weight) for m in config.llm.models], + [("local-fast", 7.0), ("local-large", 3.0)], + ) + for model in config.llm.models + config.llm.evaluator_models: + self.assertEqual(model.api_base, "http://localhost:8080/v1") + self.assertEqual(model.api_key, "local-placeholder") + prompt = PromptSampler(config.prompt).build_prompt(current_program="x = 1") + self.assertIn(data["problem_description"], prompt["system"]) + self.assertFalse(config.evaluator.cascade_evaluation) + + def test_missing_or_null_weight_defaults_to_one(self): + for models in ([{"name": "local"}], [{"name": "local", "weight": None}]): + data = experiment() + data["generation_settings"]["models"] = models + self.assertEqual(self.convert(data).llm.models[0].weight, 1.0) + + def test_description_cannot_be_mistaken_for_a_template_name(self): + data = experiment() + data["problem_description"] = "system_message" + config = self.convert(data) + self.assertEqual( + PromptSampler(config.prompt).build_prompt(current_program="x=1")["system"], + "Problem description:\nsystem_message", + ) + + def test_context_is_included_in_prompt(self): + data = experiment() + data["generation_settings"]["context"] = "Prefer short programs." + prompt = self.convert(data).prompt.system_message + self.assertIn(data["problem_description"], prompt) + self.assertIn("Prefer short programs.", prompt) + + def test_missing_or_invalid_generation_budget_never_falls_back_to_10000(self): + for budget in (None, 0, -1, True, 1.5, "10"): + data = experiment() + data["run_settings"]["max_programs"] = budget + with self.subTest(budget=budget), self.assertRaisesRegex(ValueError, "max_programs"): + self.convert(data) + + def test_missing_or_malformed_sections_raise(self): + for key in ("run_settings", "generation_settings"): + for value in (None, [], "not a mapping"): + data = experiment() + data[key] = value + with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, key): + self.convert(data) + with self.assertRaises(ValueError): + self.convert([]) + + def test_invalid_model_mixtures_raise(self): + for models in ( + None, + [], + "local", + [None], + [{}], + [{"name": ""}], + [{"name": "local", "weight": 0}], + [{"name": "local", "weight": -1}], + [{"name": "local", "weight": True}], + [{"name": "local", "weight": "1"}], + [{"name": "local", "weight": float("nan")}], + [{"name": "local", "weight": float("inf")}], + [{"name": "a", "weight": 1e308}, {"name": "b", "weight": 1e308}], + ): + data = experiment() + data["generation_settings"]["models"] = models + with self.subTest(models=models), self.assertRaises(ValueError): + self.convert(data) + + def test_language_description_endpoint_and_context_are_explicit(self): + for key, value in ( + ("program_language", "cpp"), + ("program_language", None), + ("problem_description", ""), + ("problem_description", None), + ): + data = experiment() + data[key] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + self.convert(data) + for endpoint in (None, "", " "): + with self.subTest(endpoint=endpoint), self.assertRaisesRegex(ValueError, "api_base"): + config_from_alphaevolve(experiment(), api_base=endpoint) + data = experiment() + data["generation_settings"]["context"] = 42 + with self.assertRaisesRegex(ValueError, "context"): + self.convert(data) + + def test_unsupported_settings_warn_by_path_without_leaking_values(self): + data = experiment() + data["run_settings"]["concurrency"] = 4 + data["generation_settings"]["include_full_program_in_prompt"] = False + data["generation_settings"]["models"][0]["api_key"] = "do-not-print-me" + data["project_id"] = "private-project" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + config = self.convert(data) + messages = "\n".join(str(w.message) for w in caught) + for path in ( + "run_settings.concurrency", + "generation_settings.include_full_program_in_prompt", + "generation_settings.models[0].api_key", + "project_id", + ): + self.assertIn(path, messages) + self.assertNotIn("do-not-print-me", messages) + self.assertNotIn("private-project", messages) + self.assertIsNone(config.llm.models[0].api_key) + self.assertEqual(config.evaluator.parallel_evaluations, 1) + + def test_unimplemented_wall_clock_limit_is_rejected(self): + data = experiment() + data["run_settings"]["max_duration"] = "60s" + with self.assertRaisesRegex(ValueError, "max_duration"): + self.convert(data) + + def test_sdk_null_optional_fields_do_not_warn(self): + data = experiment() + data["run_settings"]["max_duration"] = None + data["run_settings"]["concurrency"] = None + data["generation_settings"]["context"] = None + data["generation_settings"]["include_full_program_in_prompt"] = None + data["evolution_settings"] = None + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.convert(data) + self.assertEqual(caught, []) + + def test_configs_are_independent(self): + first, second = self.convert(experiment()), self.convert(experiment()) + first.llm.models[0].weight = 50 + self.assertEqual(second.llm.models[0].weight, 7) + self.assertEqual(Config().max_iterations, 10000) + + def test_imported_config_runs_with_existing_adapter_and_real_worker(self): + data = experiment() + data["run_settings"]["max_programs"] = 1 + config = self.convert(data) + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = offline_model + config.database.num_islands = 1 + + def score(candidate): + namespace = {} + exec(candidate["content"]["files"][0]["content"], namespace) + return {"scores": {"scores": [{"metric": "fitness", "score": namespace["x"]}]}} + + with tempfile.TemporaryDirectory() as directory: + result = run_evolution( + "# EVOLVE-BLOCK-START\nx = 1\n# EVOLVE-BLOCK-END\n", + adapt_evaluator(score, objective_metric="fitness"), + config=config, + output_dir=directory, + ) + self.assertEqual(result.best_score, 2) + self.assertEqual(result.best_program.generation, 1) + + +if __name__ == "__main__": + unittest.main()