From 1e520e9f7b8f3893f7dc7a51a7470ff3c7a765b4 Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Tue, 20 Oct 2026 15:59:35 -0700 Subject: [PATCH] feat(compat): import explicit AlphaEvolve dotenv settings --- examples/alphaevolve_interop/README.md | 64 ++++++- openevolve/compat/alphaevolve_env.py | 130 +++++++++++++++ pyproject.toml | 2 + tests/test_alphaevolve_env.py | 221 +++++++++++++++++++++++++ 4 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 openevolve/compat/alphaevolve_env.py create mode 100644 tests/test_alphaevolve_env.py diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 4f757cf3e..9baf323f3 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -138,9 +138,66 @@ equivalent OE controls; OE's worker count stays at its default of one unless you configure it separately. Model-specific extra fields also warn and are not forwarded. Warning messages include field paths, not their values. -This is a subset of inbound configuration support (A3 in #468), not `.env` -translation, an experiment/controller shim, or a cloud-compatible export. -Use the returned `Config` directly with the Python API. +The dictionary helper provides a subset of inbound configuration support (A3 +in #468). It does not read `.env` files, emulate the experiment/controller SDK, +or export cloud-compatible bundles. Use the returned `Config` directly with +the Python API; explicit `.env` import is described below. + +## Import an explicit example .env file + +Install the optional parser (the base OpenEvolve install is unchanged): + +```bash +python -m pip install "openevolve[alphaevolve-env]" +# From a source checkout: python -m pip install -e ".[alphaevolve-env]" +``` + +```python +from pathlib import Path +from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + +config = config_from_alphaevolve_env( + "../alphaevolve-reference/examples/tsp/example.env", + problem_description=Path( + "../alphaevolve-reference/examples/tsp/instructions.md" + ).read_text(encoding="utf-8"), + api_base="http://localhost:8000/v1", + api_key="local-placeholder", +) +# Pass config to run_evolution with the original seed and adapted evaluator. +# Importing alone does not construct clients or call the endpoint. +``` + +Only the named file is read. The helper does not find a default `.env`, read +other environment variables to fill missing settings, interpolate variable +references, change `os.environ`, or execute shell commands or the original +`run_evolution.py`. The problem text and OE endpoint/key remain explicit caller +arguments. A missing `api_key` still allows OE's usual provider behavior when +a model client is later constructed, as described above. + +| .env setting | Mapping / behavior | +| --- | --- | +| `MODEL`, optional `MODEL_WEIGHT` | Single model; missing weight becomes 1 | +| `MODEL_1..N`, optional `MODEL_i_WEIGHT` | Weighted model list in numeric index order; gaps allowed | +| `MAX_PROGRAMS_GENERATED` | Required positive integer mapped to `max_iterations` | +| `MAX_PROGRAMS_EVALUATED` | If present, must equal the generation budget; always warns that it is not a hard evaluation cap | +| Other non-null variables, including cloud credentials/endpoints and concurrency | Not imported; warnings contain names, not values | + +Either model form may be used, but not both. Empty names, orphan weights, +non-positive/non-finite weights, duplicate assignments, malformed syntax and +variable/command references in model names raise errors. Quoting, comments, +`export` prefixes and multiline quoted values are parsed with `python-dotenv`; +expansion is intentionally disabled. No model names are guessed or remapped. +Some upstream examples, including signal processing, do not declare models in +their `example.env`: add an explicit model or use the dictionary helper instead. + +An evaluation-only budget, or different generation and evaluation budgets, +is rejected rather than translated into an allegedly equivalent limit. Even +equal budgets are not equivalent: OE counts generation attempts, additionally +evaluates the seed, and may retry. This helper does not impose a strict +evaluation, request, token, time or cost limit. The existing dictionary helper +performs the final configuration validation; existing config loading and the +example command-line runner are unchanged. ## Validation @@ -150,6 +207,7 @@ callable-serialization path, without Google's SDK, keys, or network access: ```bash python -m unittest discover -s tests -p test_alphaevolve_compat.py python -m unittest discover -s tests -p test_alphaevolve_config.py +python -m unittest discover -s tests -p test_alphaevolve_env.py ``` The walkthrough additionally permits direct validation against an upstream diff --git a/openevolve/compat/alphaevolve_env.py b/openevolve/compat/alphaevolve_env.py new file mode 100644 index 000000000..7f3255e95 --- /dev/null +++ b/openevolve/compat/alphaevolve_env.py @@ -0,0 +1,130 @@ +"""Opt-in .env configuration import for local Python AlphaEvolve examples.""" + +import math +import re +import warnings +from pathlib import Path +from typing import Optional, Union + +from openevolve.compat.alphaevolve import _warn_unmapped_fields, config_from_alphaevolve +from openevolve.config import Config + + +def _read_env(path: Union[str, Path]) -> dict: + try: + from dotenv.parser import parse_stream + except ImportError: + raise ImportError( + 'AlphaEvolve .env import requires: pip install "openevolve[alphaevolve-env]"' + ) from None + + values = {} + # Use dotenv's parser bindings so malformed lines and duplicate assignments + # can fail explicitly, without interpolation, global environment changes, + # implicit file discovery, or logging source lines that may contain secrets. + with Path(path).open(encoding="utf-8-sig") as stream: + for binding in parse_stream(stream): + if binding.error: + raise ValueError(f"Invalid .env syntax near line {binding.original.line}") + key = binding.key + if key is None: + continue + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"Invalid .env variable name near line {binding.original.line}") + if key in values: + raise ValueError(f"Duplicate .env assignment near line {binding.original.line}") + values[key] = binding.value + return values + + +def _positive_integer(values: dict, key: str) -> int: + value = values.get(key) + if value is None or not re.fullmatch(r"[0-9]+", value) or int(value) < 1: + raise ValueError(f"{key} must be an explicit positive integer") + return int(value) + + +def config_from_alphaevolve_env( + env_path: Union[str, Path], + *, + problem_description: str, + api_base: str, + api_key: Optional[str] = None, +) -> Config: + """Import explicit .env model/budget settings using the dictionary converter. + + Requires the optional alphaevolve-env extra. Reads only env_path; does not + search for other files, interpolate variables, modify os.environ, execute + shell/script code, or construct clients. The caller provides the problem text + (e.g. instructions.md) and OE endpoint/key. Cloud credentials are not copied. + + Accept either MODEL with optional MODEL_WEIGHT, or MODEL_1..N with optional + MODEL_i_WEIGHT (positive indices, numeric order, gaps allowed). Mixing the + forms, missing model names, malformed/duplicate assignments, and variable or + command references in model names are rejected. Missing weights become 1. + + MAX_PROGRAMS_GENERATED is required and maps to OE generation attempts. A + present MAX_PROGRAMS_EVALUATED must be equal, otherwise import fails rather + than discarding a different limit. Even equal limits warn: OE also evaluates + the seed and may retry, so this is not a hard evaluation/request/cost cap. + Other variables, including concurrency and GCP settings, warn by name only. + """ + values = _read_env(env_path) + indices = set() + for key in values: + match = re.fullmatch(r"MODEL_([0-9]+)(?:_WEIGHT)?", key) + if match: + index = match.group(1) + if index.startswith("0"): + raise ValueError("MODEL indices must be positive integers without leading zeros") + indices.add(int(index)) + if indices and ("MODEL" in values or "MODEL_WEIGHT" in values): + raise ValueError("Use MODEL or numbered MODEL_i variables, not both") + model_keys = [f"MODEL_{i}" for i in sorted(indices)] if indices else ["MODEL"] + supported = {"MAX_PROGRAMS_GENERATED", "MAX_PROGRAMS_EVALUATED"} + models = [] + for key in model_keys: + name = values.get(key) + if not name or not name.strip() or "$" in name or "`" in name: + raise ValueError(f"{key} must contain a literal, non-empty model name") + weight_key = f"{key}_WEIGHT" + weight = 1.0 + if weight_key in values: + try: + weight = float(values[weight_key]) + except (TypeError, ValueError, OverflowError): + raise ValueError(f"{weight_key} must be finite and positive") from None + if not math.isfinite(weight) or weight <= 0: + raise ValueError(f"{weight_key} must be finite and positive") + models.append({"name": name, "weight": weight}) + supported.update({key, weight_key}) + + iterations = _positive_integer(values, "MAX_PROGRAMS_GENERATED") + if "MAX_PROGRAMS_EVALUATED" in values: + evaluated = _positive_integer(values, "MAX_PROGRAMS_EVALUATED") + if evaluated != iterations: + raise ValueError( + "MAX_PROGRAMS_EVALUATED differs from MAX_PROGRAMS_GENERATED; " + "separate generation/evaluation limits cannot be enforced by this import" + ) + + config = config_from_alphaevolve( + { + "program_language": "python", + "problem_description": problem_description, + "run_settings": {"max_programs": iterations}, + "generation_settings": {"models": models}, + }, + api_base=api_base, + api_key=api_key, + ) + if "MAX_PROGRAMS_EVALUATED" in values: + warnings.warn( + "MAX_PROGRAMS_EVALUATED is not enforced as a hard cap: " + "MAX_PROGRAMS_GENERATED maps to OE generation attempts, " + "plus a seed evaluation and possible retries", + UserWarning, + stacklevel=2, + ) + _warn_unmapped_fields(values, supported) + return config diff --git a/pyproject.toml b/pyproject.toml index 412ecc6a1..8411f9e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ ] [project.optional-dependencies] +alphaevolve-env = ["python-dotenv>=1.0,<2"] dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.21.0", @@ -29,6 +30,7 @@ dev = [ "mypy>=0.950", "requests>=2.28.0", "pre-commit>=4.5.1", + "python-dotenv>=1.0,<2", ] [tool.black] diff --git a/tests/test_alphaevolve_env.py b/tests/test_alphaevolve_env.py new file mode 100644 index 000000000..621b35561 --- /dev/null +++ b/tests/test_alphaevolve_env.py @@ -0,0 +1,221 @@ +"""Explicit, offline imports of AlphaEvolve example .env files.""" + +import builtins +import copy +import os +import tempfile +import unittest +import warnings +from pathlib import Path +from unittest.mock import patch + +from openevolve.compat.alphaevolve import config_from_alphaevolve +from openevolve.compat.alphaevolve_env import config_from_alphaevolve_env + + +class TestAlphaEvolveEnv(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) / "example.env" + + def convert(self, text, **kwargs): + self.path.write_text(text, encoding="utf-8") + return config_from_alphaevolve_env( + self.path, + problem_description="Pack circles into a unit square.", + api_base="http://localhost:8080/v1", + **kwargs, + ) + + def test_single_model_matches_dictionary_import(self): + actual = self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=3\n", api_key="local-key") + expected = config_from_alphaevolve( + { + "program_language": "python", + "problem_description": "Pack circles into a unit square.", + "run_settings": {"max_programs": 3}, + "generation_settings": {"models": [{"name": "local", "weight": 1.0}]}, + }, + api_base="http://localhost:8080/v1", + api_key="local-key", + ) + self.assertEqual(actual.to_dict(), expected.to_dict()) + + def test_numbered_models_are_sorted_numerically_with_relative_weights(self): + config = self.convert( + "MODEL_10=large\nMODEL_10_WEIGHT=0.3\nMODEL_2=fast\nMODEL_2_WEIGHT=0.7\n" + "MAX_PROGRAMS_GENERATED=10\n" + ) + self.assertEqual( + [(m.name, m.weight) for m in config.llm.models], [("fast", 0.7), ("large", 0.3)] + ) + + def test_missing_weights_default_to_one_and_single_weight_is_supported(self): + for text, weights in ( + ("MODEL=local\n", [1.0]), + ("MODEL=local\nMODEL_WEIGHT=0.5\n", [0.5]), + ("MODEL_1=a\nMODEL_2=b\n", [1.0, 1.0]), + ): + with self.subTest(text=text): + config = self.convert(text + "MAX_PROGRAMS_GENERATED=1\n") + self.assertEqual([m.weight for m in config.llm.models], weights) + + def test_dotenv_quotes_comments_export_bom_and_crlf(self): + config = self.convert( + '\ufeff# example\r\nexport MODEL = "local#tag" # inline\r\n' + "MAX_PROGRAMS_GENERATED = '2'\r\n" + ) + self.assertEqual(config.llm.models[0].name, "local#tag") + self.assertEqual(config.max_iterations, 2) + + def test_missing_empty_or_mixed_models_and_orphan_weights_raise(self): + for text in ( + "", + "MODEL=\n", + "MODEL\n", + "MODEL=' '\n", + "MODEL_WEIGHT=1\n", + "MODEL_1_WEIGHT=1\n", + "MODEL_1=a\nMODEL_2=\n", + "MODEL_0=a\n", + "MODEL_01=a\n", + "MODEL=a\nMODEL_1=b\n", + ): + with self.subTest(text=text), self.assertRaisesRegex(ValueError, "MODEL"): + self.convert(text + "MAX_PROGRAMS_GENERATED=1\n") + + def test_invalid_weights_are_rejected_without_echoing_values(self): + for weight in ("", "secret-invalid-weight", "0", "-1", "nan", "inf", "1e999"): + with self.subTest(weight=weight), self.assertRaises(ValueError) as caught: + self.convert(f"MODEL=local\nMODEL_WEIGHT={weight}\nMAX_PROGRAMS_GENERATED=1\n") + self.assertNotIn("secret-invalid-weight", str(caught.exception)) + with self.assertRaisesRegex(ValueError, "finite sum"): + self.convert( + "MODEL_1=a\nMODEL_2=b\nMODEL_1_WEIGHT=1e308\nMODEL_2_WEIGHT=1e308\nMAX_PROGRAMS_GENERATED=1\n" + ) + + def test_generation_budget_is_required_and_strictly_positive(self): + for budget in (None, "", "0", "-1", "1.5", "True", "${BUDGET}"): + text = "MODEL=local\n" + if budget is not None: + text += f"MAX_PROGRAMS_GENERATED={budget}\n" + with ( + self.subTest(budget=budget), + self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_GENERATED"), + ): + self.convert(text) + with self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_GENERATED"): + self.convert("MODEL=local\nMAX_PROGRAMS_EVALUATED=3\n") + + def test_equal_evaluation_budget_warns_that_it_is_not_a_hard_cap(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + config = self.convert( + "MODEL=local\nMAX_PROGRAMS_GENERATED=2\nMAX_PROGRAMS_EVALUATED=2\n" + ) + self.assertEqual(config.max_iterations, 2) + self.assertEqual(len(caught), 1) + self.assertIn("MAX_PROGRAMS_EVALUATED", str(caught[0].message)) + self.assertIn("seed", str(caught[0].message)) + + def test_conflicting_or_invalid_evaluation_budget_is_not_silently_dropped(self): + for budget in ("1", "3", "", "-1", "no", "1.5"): + with ( + self.subTest(budget=budget), + self.assertRaisesRegex(ValueError, "MAX_PROGRAMS_EVALUATED"), + ): + self.convert( + f"MODEL=local\nMAX_PROGRAMS_GENERATED=2\nMAX_PROGRAMS_EVALUATED={budget}\n" + ) + + def test_environment_is_neither_read_for_expansion_nor_modified(self): + with patch.dict( + os.environ, + {"MODEL": "ambient-model", "MAX_PROGRAMS_GENERATED": "999", "PRIVATE": "do-not-copy"}, + ): + before = copy.deepcopy(dict(os.environ)) + config = self.convert("MODEL=file-model\nMAX_PROGRAMS_GENERATED=2\n") + self.assertEqual(config.llm.models[0].name, "file-model") + self.assertEqual(config.max_iterations, 2) + with self.assertRaisesRegex(ValueError, "MODEL"): + self.convert("MAX_PROGRAMS_GENERATED=2\n") + for value in ("${PRIVATE}", "$PRIVATE", "$(touch marker)", "`touch marker`"): + with ( + self.subTest(value=value), + self.assertRaisesRegex(ValueError, "MODEL") as caught, + ): + self.convert(f"MODEL='{value}'\nMAX_PROGRAMS_GENERATED=1\n") + self.assertNotIn("do-not-copy", str(caught.exception)) + self.assertEqual(dict(os.environ), before) + self.assertFalse((Path(self.directory.name) / "marker").exists()) + + def test_cloud_credentials_endpoints_and_concurrency_are_not_imported(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + config = self.convert( + "MODEL=local\nMAX_PROGRAMS_GENERATED=2\nPROJECT_ID=private-project\n" + "BASE_URL=https://cloud.invalid\nOPENAI_API_KEY=secret-from-file\n" + "CONCURRENCY=4\nWORKER_CONCURRENCY=8\nPARALLEL_EVALUATION=True\n", + api_key="caller-key", + ) + messages = "\n".join(str(w.message) for w in caught) + for name in ( + "PROJECT_ID", + "BASE_URL", + "OPENAI_API_KEY", + "CONCURRENCY", + "WORKER_CONCURRENCY", + "PARALLEL_EVALUATION", + ): + self.assertIn(name, messages) + for value in ("private-project", "cloud.invalid", "secret-from-file"): + self.assertNotIn(value, messages) + self.assertEqual(config.evaluator.parallel_evaluations, 1) + for model in config.llm.models + config.llm.evaluator_models: + self.assertEqual(model.api_key, "caller-key") + self.assertEqual(model.api_base, "http://localhost:8080/v1") + + def test_missing_file_does_not_search_parent_or_default_env(self): + (Path(self.directory.name) / ".env").write_text( + "MODEL=ambient\nMAX_PROGRAMS_GENERATED=100\n" + ) + with self.assertRaises(FileNotFoundError): + config_from_alphaevolve_env( + self.path, problem_description="Problem", api_base="http://localhost/v1" + ) + + def test_duplicate_assignments_and_malformed_syntax_fail_without_values(self): + for text in ( + "MODEL=local\nMODEL=secret-duplicate\nMAX_PROGRAMS_GENERATED=1\n", + 'MODEL="secret-unclosed\nMAX_PROGRAMS_GENERATED=1\n', + "MODEL=local\nMAX_PROGRAMS_GENERATED=1\ninvalid syntax secret-malformed\n", + ): + with self.subTest(text=text), self.assertRaisesRegex(ValueError, "line") as caught: + self.convert(text) + self.assertNotIn("secret-", str(caught.exception)) + + def test_no_model_client_or_network_is_created(self): + with ( + patch("socket.socket.connect", side_effect=AssertionError("network")), + patch("openevolve.llm.openai.openai.AsyncOpenAI", side_effect=AssertionError("client")), + ): + self.assertEqual( + self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=1\n").max_iterations, 1 + ) + + def test_optional_dependency_error_is_actionable(self): + original_import = builtins.__import__ + + def without_dotenv(name, *args, **kwargs): + if name == "dotenv.parser": + raise ImportError("not installed") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=without_dotenv): + with self.assertRaisesRegex(ImportError, "openevolve\\[alphaevolve-env\\]"): + self.convert("MODEL=local\nMAX_PROGRAMS_GENERATED=1\n") + + +if __name__ == "__main__": + unittest.main()