diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 4f757cf3e..53f27734d 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -138,9 +138,120 @@ equivalent OE controls; OE's worker count stays at its default of one unless you configure it separately. Model-specific extra fields also warn and are not forwarded. Warning messages include field paths, not their values. -This is a subset of inbound configuration support (A3 in #468), not `.env` -translation, an experiment/controller shim, or a cloud-compatible export. -Use the returned `Config` directly with the Python API. +The dictionary helper provides a subset of inbound configuration support (A3 +in #468). It does not read `.env` files, emulate the experiment/controller SDK, +or export cloud-compatible bundles. Use the returned `Config` directly with +the Python API; explicit `.env` import is described below. + +## Import an explicit example .env file + +Install the optional parser (the base OpenEvolve install is unchanged): + +```bash +python -m pip install "openevolve[alphaevolve-env]" +# From a source checkout: python -m pip install -e ".[alphaevolve-env]" +``` + +```python +from pathlib import Path +from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + +config = config_from_alphaevolve_env( + "../alphaevolve-reference/examples/tsp/example.env", + problem_description=Path( + "../alphaevolve-reference/examples/tsp/instructions.md" + ).read_text(encoding="utf-8"), + api_base="http://localhost:8000/v1", + api_key="local-placeholder", +) +# Pass config to run_evolution with the original seed and adapted evaluator. +# Importing alone does not construct clients or call the endpoint. +``` + +Only the named file is read. The helper does not find a default `.env`, read +other environment variables to fill missing settings, interpolate variable +references, change `os.environ`, or execute shell commands or the original +`run_evolution.py`. The problem text and OE endpoint/key remain explicit caller +arguments. A missing `api_key` still allows OE's usual provider behavior when +a model client is later constructed, as described above. + +| .env setting | Mapping / behavior | +| --- | --- | +| `MODEL`, optional `MODEL_WEIGHT` | Single model; missing weight becomes 1 | +| `MODEL_1..N`, optional `MODEL_i_WEIGHT` | Weighted model list in numeric index order; gaps allowed | +| `MAX_PROGRAMS_GENERATED` | Required positive integer mapped to `max_iterations` | +| `MAX_PROGRAMS_EVALUATED` | If present, must equal the generation budget; always warns that it is not a hard evaluation cap | +| Other non-null variables, including cloud credentials/endpoints and concurrency | Not imported; warnings contain names, not values | + +Either model form may be used, but not both. Empty names, orphan weights, +non-positive/non-finite weights, duplicate assignments, malformed syntax and +variable/command references in model names raise errors. Quoting, comments, +`export` prefixes and multiline quoted values are parsed with `python-dotenv`; +expansion is intentionally disabled. No model names are guessed or remapped. +Some upstream examples, including signal processing, do not declare models in +their `example.env`: add an explicit model or use the dictionary helper instead. + +An evaluation-only budget, or different generation and evaluation budgets, +is rejected rather than translated into an allegedly equivalent limit. Even +equal budgets are not equivalent: OE counts generation attempts, additionally +evaluates the seed, and may retry. This helper does not impose a strict +evaluation, request, token, time or cost limit. The existing dictionary helper +performs the final configuration validation; existing config loading is +unchanged. The example runner can use either helper as described below. + +## Run with imported configuration from the command line + +The runner accepts one of `--config` (OE YAML), `--alphaevolve-config` +(experiment JSON) or `--alphaevolve-env` (an explicitly named `.env`). The two +AlphaEvolve options **only validate the configuration and evaluate the seed by +default**. Add `--evolve` to generate candidates using the selected endpoint. +The legacy `--config` option still starts evolution, with its existing default +of one iteration. + +For `.env`, install the optional parser above and prepare a copy such as +`./local.env`. Set `MODEL` or the numbered model names to models actually served +by your OE endpoint; importing a Gemini name does not make it available locally. +Provide a problem-text file explicitly rather than running the upstream +entrypoint to extract it: + +```bash +# Import/validate and evaluate the original TSP seed; no model requests. +python examples/alphaevolve_interop/run.py ../alphaevolve-reference/examples/tsp \ + --alphaevolve-env ./local.env \ + --instructions ../alphaevolve-reference/examples/tsp/instructions.md \ + --api-base http://localhost:8000/v1 + +# Explicit model-driven evolution; may incur charges at the selected endpoint. +python examples/alphaevolve_interop/run.py ../alphaevolve-reference/examples/tsp \ + --alphaevolve-env ./local.env \ + --instructions ../alphaevolve-reference/examples/tsp/instructions.md \ + --api-base http://localhost:8000/v1 \ + --evolve --iterations 1 --output-dir ./interop-output +``` + +Alternatively, save an experiment dictionary (including `program_language`, +`problem_description`, `run_settings` and `generation_settings`) as JSON: + +```bash +python examples/alphaevolve_interop/run.py ../alphaevolve-reference/examples/circle_packing \ + --alphaevolve-config ./experiment.json --api-base http://localhost:8000/v1 \ + --evolve --iterations 1 --output-dir ./interop-output +``` + +Both imports require an explicit `--api-base`. To select a key without putting +its value in command arguments, add `--api-key-env OE_API_KEY` after setting +that environment variable yourself. An explicitly selected missing/empty key +variable is an error. If this option is omitted, the existing OE provider's +key behavior (including `OPENAI_API_KEY` fallback) applies when a client is +constructed. Nothing is copied from Google credential fields in the source. + +With `--evolve`, omitting `--iterations` uses the imported generation budget. +An explicit positive `--iterations` may only reduce that budget. As above, +generation attempts are not a strict request/evaluation/cost cap. `--iterations` +without an evolution mode is rejected rather than ignored. `--instructions` +is required only for `.env`; JSON already contains the problem description. +Invalid option combinations and import errors are reported before loading +the example evaluator. The original cloud `run_evolution.py` is not imported. ## Validation @@ -150,6 +261,8 @@ callable-serialization path, without Google's SDK, keys, or network access: ```bash python -m unittest discover -s tests -p test_alphaevolve_compat.py python -m unittest discover -s tests -p test_alphaevolve_config.py +python -m unittest discover -s tests -p test_alphaevolve_env.py +python -m unittest discover -s tests -p test_alphaevolve_runner.py ``` The walkthrough additionally permits direct validation against an upstream diff --git a/examples/alphaevolve_interop/run.py b/examples/alphaevolve_interop/run.py index 3e7bdec00..3575a7bf2 100644 --- a/examples/alphaevolve_interop/run.py +++ b/examples/alphaevolve_interop/run.py @@ -1,12 +1,13 @@ -"""Evaluate an unchanged AlphaEvolve example; optionally evolve it with an OE config.""" +"""Evaluate an unchanged AlphaEvolve example; explicitly opt in to model-driven evolution.""" import argparse import importlib.util import json +import os from pathlib import Path from openevolve import run_evolution -from openevolve.compat.alphaevolve import adapt_evaluator +from openevolve.compat.alphaevolve import adapt_evaluator, config_from_alphaevolve EXAMPLES = { "circle_packing": ("circle_packing_evaluation", "sum_of_radii"), @@ -30,25 +31,92 @@ def load_evaluator(example_dir): return adapt_evaluator(getattr(module, function_name), objective_metric=objective) -def main(): +def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("example_dir", type=Path, help="Path to the upstream example directory") + sources = parser.add_mutually_exclusive_group() + sources.add_argument( + "--config", type=Path, help="OE YAML config; starts evolution (legacy mode)" + ) + sources.add_argument("--alphaevolve-config", type=Path, help="AlphaEvolve experiment JSON") + sources.add_argument("--alphaevolve-env", type=Path, help="Explicit AlphaEvolve .env file") + parser.add_argument( + "--instructions", type=Path, help="Problem text file; required with --alphaevolve-env" + ) + parser.add_argument("--api-base", help="Explicit OE model endpoint for AlphaEvolve imports") + parser.add_argument( + "--api-key-env", help="Read the OE API key from this named environment variable" + ) parser.add_argument( - "--config", type=Path, help="OE YAML config; omit to evaluate only (no LLM)" + "--evolve", + action="store_true", + help="Run imported AlphaEvolve config using the chosen model (may incur charges)", ) parser.add_argument( - "--iterations", type=int, default=1, help="Evolution iterations with --config" + "--iterations", + type=int, + help="Positive generation attempts; may only reduce an imported budget (legacy --config default: 1)", ) parser.add_argument("--output-dir", type=Path, default=Path("alphaevolve_output")) - args = parser.parse_args() + args = parser.parse_args(argv) example_dir = args.example_dir.resolve() if example_dir.name not in EXAMPLES: parser.error(f"Supported example names: {', '.join(EXAMPLES)}") - if args.iterations < 1: + imported = args.alphaevolve_config is not None or args.alphaevolve_env is not None + evolving = args.config is not None or args.evolve + if args.evolve and not (args.config or imported): + parser.error("--evolve requires a configuration source") + if args.iterations is not None and args.iterations < 1: parser.error("--iterations must be positive") + if args.iterations is not None and not evolving: + parser.error("--iterations requires --evolve or legacy --config") + if not imported and ( + args.api_base is not None or args.api_key_env is not None or args.instructions is not None + ): + parser.error("--api-base, --api-key-env and --instructions require an AlphaEvolve source") + if imported and not args.api_base: + parser.error("--api-base is required for AlphaEvolve imports") + if args.alphaevolve_env is not None and args.instructions is None: + parser.error("--instructions is required with --alphaevolve-env") + if args.instructions is not None and args.alphaevolve_env is None: + parser.error( + "--instructions is only used with --alphaevolve-env; JSON includes problem_description" + ) + + config = args.config + if imported: + api_key = None + if args.api_key_env is not None: + api_key = os.environ.get(args.api_key_env) + if not api_key or not api_key.strip(): + parser.error("--api-key-env must name a non-empty environment variable") + try: + if args.alphaevolve_env is not None: + from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + + config = config_from_alphaevolve_env( + args.alphaevolve_env, + problem_description=args.instructions.read_text(encoding="utf-8"), + api_base=args.api_base, + api_key=api_key, + ) + else: + config = config_from_alphaevolve( + json.loads(args.alphaevolve_config.read_text(encoding="utf-8-sig")), + api_base=args.api_base, + api_key=api_key, + ) + except (ValueError, OSError, ImportError) as exc: + parser.error(str(exc)) + if args.iterations is not None and args.iterations > config.max_iterations: + parser.error("--iterations cannot exceed the imported generation budget") + program_path = example_dir / "src" / "program.py" + for path in (program_path, example_dir / "src" / "evaluate.py"): + if not path.is_file(): + parser.error(f"Required example file not found: {path}") evaluator = load_evaluator(example_dir) - if args.config is None: + if not evolving: result = evaluator(program_path) print(json.dumps({"metrics": result.metrics, "artifacts": result.artifacts}, indent=2)) return @@ -56,8 +124,8 @@ def main(): result = run_evolution( initial_program=program_path, evaluator=evaluator, - config=args.config, - iterations=args.iterations, + config=config, + iterations=args.iterations if imported else (args.iterations or 1), output_dir=str(args.output_dir), cleanup=True, ) diff --git a/openevolve/compat/alphaevolve_env.py b/openevolve/compat/alphaevolve_env.py new file mode 100644 index 000000000..7f3255e95 --- /dev/null +++ b/openevolve/compat/alphaevolve_env.py @@ -0,0 +1,130 @@ +"""Opt-in .env configuration import for local Python AlphaEvolve examples.""" + +import math +import re +import warnings +from pathlib import Path +from typing import Optional, Union + +from openevolve.compat.alphaevolve import _warn_unmapped_fields, config_from_alphaevolve +from openevolve.config import Config + + +def _read_env(path: Union[str, Path]) -> dict: + try: + from dotenv.parser import parse_stream + except ImportError: + raise ImportError( + 'AlphaEvolve .env import requires: pip install "openevolve[alphaevolve-env]"' + ) from None + + values = {} + # Use dotenv's parser bindings so malformed lines and duplicate assignments + # can fail explicitly, without interpolation, global environment changes, + # implicit file discovery, or logging source lines that may contain secrets. + with Path(path).open(encoding="utf-8-sig") as stream: + for binding in parse_stream(stream): + if binding.error: + raise ValueError(f"Invalid .env syntax near line {binding.original.line}") + key = binding.key + if key is None: + continue + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"Invalid .env variable name near line {binding.original.line}") + if key in values: + raise ValueError(f"Duplicate .env assignment near line {binding.original.line}") + values[key] = binding.value + return values + + +def _positive_integer(values: dict, key: str) -> int: + value = values.get(key) + if value is None or not re.fullmatch(r"[0-9]+", value) or int(value) < 1: + raise ValueError(f"{key} must be an explicit positive integer") + return int(value) + + +def config_from_alphaevolve_env( + env_path: Union[str, Path], + *, + problem_description: str, + api_base: str, + api_key: Optional[str] = None, +) -> Config: + """Import explicit .env model/budget settings using the dictionary converter. + + Requires the optional alphaevolve-env extra. Reads only env_path; does not + search for other files, interpolate variables, modify os.environ, execute + shell/script code, or construct clients. The caller provides the problem text + (e.g. instructions.md) and OE endpoint/key. Cloud credentials are not copied. + + Accept either MODEL with optional MODEL_WEIGHT, or MODEL_1..N with optional + MODEL_i_WEIGHT (positive indices, numeric order, gaps allowed). Mixing the + forms, missing model names, malformed/duplicate assignments, and variable or + command references in model names are rejected. Missing weights become 1. + + MAX_PROGRAMS_GENERATED is required and maps to OE generation attempts. A + present MAX_PROGRAMS_EVALUATED must be equal, otherwise import fails rather + than discarding a different limit. Even equal limits warn: OE also evaluates + the seed and may retry, so this is not a hard evaluation/request/cost cap. + Other variables, including concurrency and GCP settings, warn by name only. + """ + values = _read_env(env_path) + indices = set() + for key in values: + match = re.fullmatch(r"MODEL_([0-9]+)(?:_WEIGHT)?", key) + if match: + index = match.group(1) + if index.startswith("0"): + raise ValueError("MODEL indices must be positive integers without leading zeros") + indices.add(int(index)) + if indices and ("MODEL" in values or "MODEL_WEIGHT" in values): + raise ValueError("Use MODEL or numbered MODEL_i variables, not both") + model_keys = [f"MODEL_{i}" for i in sorted(indices)] if indices else ["MODEL"] + supported = {"MAX_PROGRAMS_GENERATED", "MAX_PROGRAMS_EVALUATED"} + models = [] + for key in model_keys: + name = values.get(key) + if not name or not name.strip() or "$" in name or "`" in name: + raise ValueError(f"{key} must contain a literal, non-empty model name") + weight_key = f"{key}_WEIGHT" + weight = 1.0 + if weight_key in values: + try: + weight = float(values[weight_key]) + except (TypeError, ValueError, OverflowError): + raise ValueError(f"{weight_key} must be finite and positive") from None + if not math.isfinite(weight) or weight <= 0: + raise ValueError(f"{weight_key} must be finite and positive") + models.append({"name": name, "weight": weight}) + supported.update({key, weight_key}) + + iterations = _positive_integer(values, "MAX_PROGRAMS_GENERATED") + if "MAX_PROGRAMS_EVALUATED" in values: + evaluated = _positive_integer(values, "MAX_PROGRAMS_EVALUATED") + if evaluated != iterations: + raise ValueError( + "MAX_PROGRAMS_EVALUATED differs from MAX_PROGRAMS_GENERATED; " + "separate generation/evaluation limits cannot be enforced by this import" + ) + + config = config_from_alphaevolve( + { + "program_language": "python", + "problem_description": problem_description, + "run_settings": {"max_programs": iterations}, + "generation_settings": {"models": models}, + }, + api_base=api_base, + api_key=api_key, + ) + if "MAX_PROGRAMS_EVALUATED" in values: + warnings.warn( + "MAX_PROGRAMS_EVALUATED is not enforced as a hard cap: " + "MAX_PROGRAMS_GENERATED maps to OE generation attempts, " + "plus a seed evaluation and possible retries", + UserWarning, + stacklevel=2, + ) + _warn_unmapped_fields(values, supported) + return config diff --git a/pyproject.toml b/pyproject.toml index 412ecc6a1..8411f9e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ ] [project.optional-dependencies] +alphaevolve-env = ["python-dotenv>=1.0,<2"] dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.21.0", @@ -29,6 +30,7 @@ dev = [ "mypy>=0.950", "requests>=2.28.0", "pre-commit>=4.5.1", + "python-dotenv>=1.0,<2", ] [tool.black] diff --git a/tests/test_alphaevolve_env.py b/tests/test_alphaevolve_env.py new file mode 100644 index 000000000..621b35561 --- /dev/null +++ b/tests/test_alphaevolve_env.py @@ -0,0 +1,221 @@ +"""Explicit, offline imports of AlphaEvolve example .env files.""" + +import builtins +import copy +import os +import tempfile +import unittest +import warnings +from pathlib import Path +from unittest.mock import patch + +from openevolve.compat.alphaevolve import config_from_alphaevolve +from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + + +class TestAlphaEvolveEnv(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) / "example.env" + + def convert(self, text, **kwargs): + self.path.write_text(text, encoding="utf-8") + return config_from_alphaevolve_env( + self.path, + problem_description="Pack circles into a unit square.", + api_base="http://localhost:8080/v1", + **kwargs, + ) + + def test_single_model_matches_dictionary_import(self): + actual = self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=3\n", api_key="local-key") + expected = config_from_alphaevolve( + { + "program_language": "python", + "problem_description": "Pack circles into a unit square.", + "run_settings": {"max_programs": 3}, + "generation_settings": {"models": [{"name": "local", "weight": 1.0}]}, + }, + api_base="http://localhost:8080/v1", + api_key="local-key", + ) + self.assertEqual(actual.to_dict(), expected.to_dict()) + + def test_numbered_models_are_sorted_numerically_with_relative_weights(self): + config = self.convert( + "MODEL_10=large\nMODEL_10_WEIGHT=0.3\nMODEL_2=fast\nMODEL_2_WEIGHT=0.7\n" + "MAX_PROGRAMS_GENERATED=10\n" + ) + self.assertEqual( + [(m.name, m.weight) for m in config.llm.models], [("fast", 0.7), ("large", 0.3)] + ) + + def test_missing_weights_default_to_one_and_single_weight_is_supported(self): + for text, weights in ( + ("MODEL=local\n", [1.0]), + ("MODEL=local\nMODEL_WEIGHT=0.5\n", [0.5]), + ("MODEL_1=a\nMODEL_2=b\n", [1.0, 1.0]), + ): + with self.subTest(text=text): + config = self.convert(text + "MAX_PROGRAMS_GENERATED=1\n") + self.assertEqual([m.weight for m in config.llm.models], weights) + + def test_dotenv_quotes_comments_export_bom_and_crlf(self): + config = self.convert( + '\ufeff# example\r\nexport MODEL = "local#tag" # inline\r\n' + "MAX_PROGRAMS_GENERATED = '2'\r\n" + ) + self.assertEqual(config.llm.models[0].name, "local#tag") + self.assertEqual(config.max_iterations, 2) + + def test_missing_empty_or_mixed_models_and_orphan_weights_raise(self): + for text in ( + "", + "MODEL=\n", + "MODEL\n", + "MODEL=' '\n", + "MODEL_WEIGHT=1\n", + "MODEL_1_WEIGHT=1\n", + "MODEL_1=a\nMODEL_2=\n", + "MODEL_0=a\n", + "MODEL_01=a\n", + "MODEL=a\nMODEL_1=b\n", + ): + with self.subTest(text=text), self.assertRaisesRegex(ValueError, "MODEL"): + self.convert(text + "MAX_PROGRAMS_GENERATED=1\n") + + def test_invalid_weights_are_rejected_without_echoing_values(self): + for weight in ("", "secret-invalid-weight", "0", "-1", "nan", "inf", "1e999"): + with self.subTest(weight=weight), self.assertRaises(ValueError) as caught: + self.convert(f"MODEL=local\nMODEL_WEIGHT={weight}\nMAX_PROGRAMS_GENERATED=1\n") + self.assertNotIn("secret-invalid-weight", str(caught.exception)) + with self.assertRaisesRegex(ValueError, "finite sum"): + self.convert( + "MODEL_1=a\nMODEL_2=b\nMODEL_1_WEIGHT=1e308\nMODEL_2_WEIGHT=1e308\nMAX_PROGRAMS_GENERATED=1\n" + ) + + def test_generation_budget_is_required_and_strictly_positive(self): + for budget in (None, "", "0", "-1", "1.5", "True", "${BUDGET}"): + text = "MODEL=local\n" + if budget is not None: + text += f"MAX_PROGRAMS_GENERATED={budget}\n" + with ( + self.subTest(budget=budget), + self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_GENERATED"), + ): + self.convert(text) + with self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_GENERATED"): + self.convert("MODEL=local\nMAX_PROGRAMS_EVALUATED=3\n") + + def test_equal_evaluation_budget_warns_that_it_is_not_a_hard_cap(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + config = self.convert( + "MODEL=local\nMAX_PROGRAMS_GENERATED=2\nMAX_PROGRAMS_EVALUATED=2\n" + ) + self.assertEqual(config.max_iterations, 2) + self.assertEqual(len(caught), 1) + self.assertIn("MAX_PROGRAMS_EVALUATED", str(caught[0].message)) + self.assertIn("seed", str(caught[0].message)) + + def test_conflicting_or_invalid_evaluation_budget_is_not_silently_dropped(self): + for budget in ("1", "3", "", "-1", "no", "1.5"): + with ( + self.subTest(budget=budget), + self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_EVALUATED"), + ): + self.convert( + f"MODEL=local\nMAX_PROGRAMS_GENERATED=2\nMAX_PROGRAMS_EVALUATED={budget}\n" + ) + + def test_environment_is_neither_read_for_expansion_nor_modified(self): + with patch.dict( + os.environ, + {"MODEL": "ambient-model", "MAX_PROGRAMS_GENERATED": "999", "PRIVATE": "do-not-copy"}, + ): + before = copy.deepcopy(dict(os.environ)) + config = self.convert("MODEL=file-model\nMAX_PROGRAMS_GENERATED=2\n") + self.assertEqual(config.llm.models[0].name, "file-model") + self.assertEqual(config.max_iterations, 2) + with self.assertRaisesRegex(ValueError, "MODEL"): + self.convert("MAX_PROGRAMS_GENERATED=2\n") + for value in ("${PRIVATE}", "$PRIVATE", "$(touch marker)", "`touch marker`"): + with ( + self.subTest(value=value), + self.assertRaisesRegex(ValueError, "MODEL") as caught, + ): + self.convert(f"MODEL='{value}'\nMAX_PROGRAMS_GENERATED=1\n") + self.assertNotIn("do-not-copy", str(caught.exception)) + self.assertEqual(dict(os.environ), before) + self.assertFalse((Path(self.directory.name) / "marker").exists()) + + def test_cloud_credentials_endpoints_and_concurrency_are_not_imported(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + config = self.convert( + "MODEL=local\nMAX_PROGRAMS_GENERATED=2\nPROJECT_ID=private-project\n" + "BASE_URL=https://cloud.invalid\nOPENAI_API_KEY=secret-from-file\n" + "CONCURRENCY=4\nWORKER_CONCURRENCY=8\nPARALLEL_EVALUATION=True\n", + api_key="caller-key", + ) + messages = "\n".join(str(w.message) for w in caught) + for name in ( + "PROJECT_ID", + "BASE_URL", + "OPENAI_API_KEY", + "CONCURRENCY", + "WORKER_CONCURRENCY", + "PARALLEL_EVALUATION", + ): + self.assertIn(name, messages) + for value in ("private-project", "cloud.invalid", "secret-from-file"): + self.assertNotIn(value, messages) + self.assertEqual(config.evaluator.parallel_evaluations, 1) + for model in config.llm.models + config.llm.evaluator_models: + self.assertEqual(model.api_key, "caller-key") + self.assertEqual(model.api_base, "http://localhost:8080/v1") + + def test_missing_file_does_not_search_parent_or_default_env(self): + (Path(self.directory.name) / ".env").write_text( + "MODEL=ambient\nMAX_PROGRAMS_GENERATED=100\n" + ) + with self.assertRaises(FileNotFoundError): + config_from_alphaevolve_env( + self.path, problem_description="Problem", api_base="http://localhost/v1" + ) + + def test_duplicate_assignments_and_malformed_syntax_fail_without_values(self): + for text in ( + "MODEL=local\nMODEL=secret-duplicate\nMAX_PROGRAMS_GENERATED=1\n", + 'MODEL="secret-unclosed\nMAX_PROGRAMS_GENERATED=1\n', + "MODEL=local\nMAX_PROGRAMS_GENERATED=1\ninvalid syntax secret-malformed\n", + ): + with self.subTest(text=text), self.assertRaisesRegex(ValueError, "line") as caught: + self.convert(text) + self.assertNotIn("secret-", str(caught.exception)) + + def test_no_model_client_or_network_is_created(self): + with ( + patch("socket.socket.connect", side_effect=AssertionError("network")), + patch("openevolve.llm.openai.openai.AsyncOpenAI", side_effect=AssertionError("client")), + ): + self.assertEqual( + self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=1\n").max_iterations, 1 + ) + + def test_optional_dependency_error_is_actionable(self): + original_import = builtins.__import__ + + def without_dotenv(name, *args, **kwargs): + if name == "dotenv.parser": + raise ImportError("not installed") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=without_dotenv): + with self.assertRaisesRegex(ImportError, "openevolve\\[alphaevolve-env\\]"): + self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=1\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_alphaevolve_runner.py b/tests/test_alphaevolve_runner.py new file mode 100644 index 000000000..34da7c2da --- /dev/null +++ b/tests/test_alphaevolve_runner.py @@ -0,0 +1,281 @@ +"""Exercise the example CLI with local fixtures and an offline evolution worker.""" + +import contextlib +import importlib.util +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + +RUNNER = Path(__file__).resolve().parents[1] / "examples/alphaevolve_interop/run.py" +spec = importlib.util.spec_from_file_location("_alphaevolve_runner", RUNNER) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class OfflineModel: + def __init__(self, config): + self.model = config.name + + async def generate_with_context(self, system_message, messages, **kwargs): + assert "Pack circles" in system_message + return "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE" + + +def offline_model(config): + return OfflineModel(config) + + +class TestAlphaEvolveRunner(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + for name, function, metric in ( + ("circle_packing", "circle_packing_evaluation", "sum_of_radii"), + ("tsp", "tsp_evaluation_function", "neg_tour_length"), + ("signal_processing", "signal_processing_evaluation", "overall_score"), + ): + source = self.root / name / "src" + source.mkdir(parents=True) + (source / "program.py").write_text("# EVOLVE-BLOCK-START\nx = 1\n# EVOLVE-BLOCK-END\n") + (source / "evaluate.py").write_text( + f"def {function}(candidate):\n" + " namespace = {}\n" + " exec(candidate['content']['files'][0]['content'], namespace)\n" + f" return {{'scores': {{'scores': [{{'metric': '{metric}', 'score': namespace['x']}}]}}}}\n" + ) + # Loading an upstream cloud entrypoint would fail this test. + (source / "run_evolution.py").write_text("raise AssertionError('cloud entrypoint')\n") + self.example = self.root / "circle_packing" + self.instructions = self.root / "instructions.md" + self.instructions.write_text("Pack circles into a unit square.") + self.env = self.root / "example.env" + self.env.write_text("MODEL=offline-model\nMAX_PROGRAMS_GENERATED=3\n") + self.payload = { + "program_language": "python", + "problem_description": self.instructions.read_text(), + "run_settings": {"max_programs": 3}, + "generation_settings": {"models": [{"name": "offline-model"}]}, + } + self.config_json = self.root / "experiment.json" + self.config_json.write_text(json.dumps(self.payload)) + self.json_args = [ + "--alphaevolve-config", + str(self.config_json), + "--api-base", + "http://localhost:8080/v1", + ] + self.env_args = [ + "--alphaevolve-env", + str(self.env), + "--instructions", + str(self.instructions), + "--api-base", + "http://localhost:8080/v1", + ] + + def invoke(self, args): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + runner.main([str(self.example), *args]) + return json.loads(stdout.getvalue()) + + def assert_invalid(self, args): + stderr = io.StringIO() + with ( + contextlib.redirect_stderr(stderr), + patch.object( + runner, + "load_evaluator", + side_effect=AssertionError("must validate before loading evaluator"), + ), + patch.object(runner, "run_evolution") as evolve, + ): + with self.assertRaises(SystemExit) as caught: + self.invoke(args) + self.assertEqual(caught.exception.code, 2) + evolve.assert_not_called() + self.assertNotIn("Traceback", stderr.getvalue()) + return stderr.getvalue() + + def test_no_config_and_imported_configs_default_to_seed_only(self): + for args in ([], self.json_args, self.env_args): + with ( + self.subTest(args=args), + patch.object(runner, "run_evolution") as evolve, + patch( + "openevolve.llm.openai.openai.AsyncOpenAI", side_effect=AssertionError("client") + ), + patch("socket.socket.connect", side_effect=AssertionError("network")), + ): + self.assertEqual(self.invoke(args)["metrics"]["sum_of_radii"], 1) + evolve.assert_not_called() + + def test_explicit_json_evolution_uses_imported_budget(self): + with patch.object( + runner, + "run_evolution", + return_value=SimpleNamespace(metrics={"combined_score": 2}, best_score=2), + ) as evolve: + result = self.invoke([*self.json_args, "--evolve"]) + config = evolve.call_args.kwargs["config"] + self.assertEqual(config.max_iterations, 3) + self.assertEqual(config.llm.models[0].name, "offline-model") + self.assertIsNone(evolve.call_args.kwargs["iterations"]) + self.assertEqual(result["best_score"], 2) + + def test_env_evolution_can_reduce_budget_and_select_key_by_variable_name(self): + with ( + patch.dict(os.environ, {"OE_INTEROP_TEST_KEY": "do-not-print-key"}), + patch.object( + runner, + "run_evolution", + return_value=SimpleNamespace(metrics={"combined_score": 2}, best_score=2), + ) as evolve, + ): + result = self.invoke( + [ + *self.env_args, + "--evolve", + "--iterations", + "2", + "--api-key-env", + "OE_INTEROP_TEST_KEY", + ] + ) + kwargs = evolve.call_args.kwargs + self.assertEqual(kwargs["iterations"], 2) + self.assertEqual(kwargs["config"].max_iterations, 3) + self.assertEqual(kwargs["config"].llm.models[0].api_key, "do-not-print-key") + self.assertNotIn("do-not-print-key", json.dumps(result)) + + def test_legacy_yaml_mode_keeps_one_iteration_default_and_explicit_override(self): + config_path = self.root / "oe.yaml" + for extra, expected in (([], 1), (["--iterations", "2"], 2)): + with ( + self.subTest(extra=extra), + patch.object( + runner, "run_evolution", return_value=SimpleNamespace(metrics={}, best_score=1) + ) as evolve, + ): + self.invoke(["--config", str(config_path), *extra]) + self.assertEqual(evolve.call_args.kwargs["config"], config_path) + self.assertEqual(evolve.call_args.kwargs["iterations"], expected) + + def test_invalid_flag_combinations_fail_before_loading_example_code(self): + for args in ( + ["--evolve"], + ["--iterations", "1"], + [*self.json_args, "--iterations", "1"], + ["--api-base", "http://localhost/v1"], + ["--api-key-env", "ANY_KEY"], + ["--instructions", str(self.instructions)], + ["--alphaevolve-config", str(self.config_json)], + ["--alphaevolve-env", str(self.env), "--api-base", "http://localhost/v1"], + [*self.json_args, "--instructions", str(self.instructions)], + [*self.json_args, "--config", "oe.yaml"], + [*self.json_args, "--alphaevolve-env", str(self.env)], + ["--config", "oe.yaml", "--api-base", "http://localhost/v1"], + ): + with self.subTest(args=args): + self.assert_invalid(args) + + def test_iteration_override_must_be_positive_and_cannot_increase_imported_budget(self): + for source in (self.json_args, self.env_args): + for count in ("0", "-1", "4", "not-an-int"): + with self.subTest(source=source, count=count): + self.assert_invalid([*source, "--evolve", "--iterations", count]) + + def test_malformed_missing_or_invalid_config_files_are_usage_errors(self): + for text in ("not JSON", "[]", "{}"): + self.config_json.write_text(text) + with self.subTest(text=text): + self.assert_invalid(self.json_args) + self.config_json.unlink() + self.assert_invalid(self.json_args) + self.env.write_text("MODEL=x\nMAX_PROGRAMS_GENERATED=2\nMAX_PROGRAMS_EVALUATED=1\n") + self.assertIn("MAX_PROGRAMS_EVALUATED", self.assert_invalid(self.env_args)) + self.instructions.unlink() + self.assert_invalid(self.env_args) + + def test_explicit_missing_or_blank_key_variable_is_not_replaced_by_ambient_key(self): + for value in (None, "", " "): + with ( + self.subTest(value=value), + patch.dict(os.environ, {"OPENAI_API_KEY": "ambient-key"}), + ): + if value is None: + os.environ.pop("OE_INTEROP_TEST_KEY", None) + else: + os.environ["OE_INTEROP_TEST_KEY"] = value + message = self.assert_invalid( + [*self.json_args, "--api-key-env", "OE_INTEROP_TEST_KEY"] + ) + self.assertIn("--api-key-env", message) + self.assertNotIn("ambient-key", message) + + def test_missing_optional_parser_is_actionable_and_json_seed_still_works(self): + with patch( + "openevolve.compat.alphaevolve_env._read_env", + side_effect=ImportError('pip install "openevolve[alphaevolve-env]"'), + ): + self.assertIn("openevolve[alphaevolve-env]", self.assert_invalid(self.env_args)) + self.assertEqual(self.invoke(self.json_args)["metrics"]["sum_of_radii"], 1) + + def test_missing_seed_or_evaluator_is_reported_before_import(self): + (self.example / "src/program.py").unlink() + self.assertIn("Required example file", self.assert_invalid([])) + + def test_imported_env_runs_one_generation_through_real_controller_and_worker(self): + self.env.write_text("MODEL=offline-model\nMAX_PROGRAMS_GENERATED=1\n") + + def offline_config(*args, **kwargs): + config = config_from_alphaevolve_env(*args, **kwargs) + config.database.num_islands = 1 + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = offline_model + return config + + with ( + patch( + "openevolve.compat.alphaevolve_env.config_from_alphaevolve_env", + side_effect=offline_config, + ), + patch("socket.socket.connect", side_effect=AssertionError("network")), + ): + result = self.invoke( + [*self.env_args, "--evolve", "--output-dir", str(self.root / "output")] + ) + self.assertEqual(result["metrics"]["sum_of_radii"], 2) + self.assertEqual(result["best_score"], 2) + self.assertIn("x = 1", (self.example / "src/program.py").read_text()) + self.assertTrue((self.root / "output").is_dir()) + + def test_real_cli_subprocess_evaluates_each_supported_fixture(self): + for name, metric in ( + ("circle_packing", "sum_of_radii"), + ("tsp", "neg_tour_length"), + ("signal_processing", "overall_score"), + ): + with self.subTest(name=name): + result = subprocess.run( + [sys.executable, str(RUNNER), str(self.root / name)], + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["metrics"][metric], 1) + + +if __name__ == "__main__": + unittest.main()