diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 243174e30..b47ca1c1c 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -205,6 +205,96 @@ This closes the local evaluator-type/import gap, not the complete SDK shim or the original-entrypoint acceptance criterion for #468. It neither enables cloud execution nor changes the facade's budget, concurrency or best-only query limits. +## Run the original circle-packing or TSP entrypoint + +`run_entrypoint.py` executes the upstream `src/run_evolution.py`'s original +`main()` for **circle_packing** and **tsp** with explicit local connection/model +settings. It stages byte-identical copies of `src/` and optional +`instructions.md`, imports their relative helpers under temporary local aliases, +and swaps imported client/experiment factories and configuration globals. It +does not rewrite the source, replace the evaluator, or contact Google's SDK. + +From an installed OE checkout, with a reviewed upstream checkout as above: + +```bash +python -m pip install numpy scipy 'matplotlib>=3.9.1,<3.11' nest-asyncio python-dotenv + +# Run the original TSP main, including its seed scoring and report generation. +# No generation requests. The output directory must be new and outside the example. +MPLBACKEND=Agg python examples/alphaevolve_interop/run_entrypoint.py \ + ../alphaevolve-reference/examples/tsp \ + --api-base http://localhost:8000/v1 --model my-local-model \ + --output-dir ./tsp-local-seed + +# Explicitly opt in to one model-driven generation attempt; may incur charges. +MPLBACKEND=Agg python examples/alphaevolve_interop/run_entrypoint.py \ + ../alphaevolve-reference/examples/circle_packing \ + --api-base http://localhost:8000/v1 --model my-local-model \ + --evolve --iterations 1 --output-dir ./circle-local-run +``` + +The endpoint/model are required even for the default seed-only run; their live +availability is not checked until generation is requested. `--evolve` defaults +to one generation attempt. Explicit positive `--iterations` requires `--evolve`. +These are OE generation attempts, not hard evaluation/request/time/cost caps; +seed evaluation, retries and the original reporting code's own evaluations are +additional. An upstream entrypoint can score its seed before OE scores it again. +This runner does not execute signal processing or remote/cloud-evaluation +examples; signal processing still needs its cloud-specific query/controller +calls adapted in a separate step. + +Only a named `--api-key-env VARIABLE_NAME` selects a key, which must be non-empty. +Without it, an explicit local placeholder is used, including for provider-client +construction in seed-only mode. The runner does **not** fall back to +`OPENAI_API_KEY` or an example `.env` for credentials. Authenticated endpoints +need an explicitly selected key before using `--evolve`. Google connection +arguments in the upstream main are accepted by the existing facade and warn +that they are ignored. + +The runner disables `dotenv.load_dotenv` during entrypoint import and temporarily +supplies the selected model, equal positive source budgets, synchronous +evaluation and concurrency-one configuration. Import-time environment changes +are restored afterwards. It then sets the corresponding entrypoint globals +explicitly, including zero local generation iterations for a seed-only run. This +does not equate cloud generation concurrency with OE worker concurrency: the +facade keeps its existing single-worker behavior and unsupported-field warnings. +Missing expected configuration names cause a layout-drift error rather than a +guessed adaptation. + +`.env*` files and bytecode caches are excluded from the staged `src/` copy; +symlinks in the selected source tree or instructions are rejected. Original +source files remain untouched, and reports operate on the staged copy. A new +`--output-dir` retains copied sources, `oe-output/`, and upstream report artifacts, +including partial output after failure. Existing directories/files/symlinks are +never reused; the parent directory must already exist. Without `--output-dir`, +the temporary run and reports are removed on success or failure. + +The final stdout is an OE metrics/best-score JSON summary; Python-level upstream +prints are redirected to stderr. Upstream report/plot routines still execute; +`MPLBACKEND=Agg` avoids interactive plot windows. Reports receive **only the +facade's best result**, not the complete archive, so their history charts and +reported program-list counts are not full-run evaluation statistics. +The reviewed circle-packing plot helper uses `matplotlib.cm.get_cmap`, which +[Matplotlib removed in 3.11](https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.11.0.html#matplotlib-cm-get-cmap). +The optional dependency range above retains that upstream plotting API; it is +not a restriction on OE's base dependencies. This runner does not patch plotting +libraries or rewrite upstream visualization code. + +Use a fresh, single-threaded main process outside an active asyncio loop. +Temporary package and SDK aliases are removed before `main()` so evaluator +callables serialize by value to workers. Already-loaded cloud SDK namespaces +are refused. Trusted source code, including import-time code and plotting +helpers, executes with normal process privileges; copying and alias cleanup +provide no sandbox or general rollback of external side effects. Dynamic +relative imports after initial loading are outside this reviewed subset. + +This walkthrough covers two original-entrypoint paths toward #468. It is not a +full SDK replacement, managed-service validation or completion of the issue. + +```bash +python -m unittest discover -s tests -p 'test_alphaevolve_entrypoint.py' -v +``` + ## Evaluate an upstream seed locally From an installed OpenEvolve checkout: diff --git a/examples/alphaevolve_interop/run_entrypoint.py b/examples/alphaevolve_interop/run_entrypoint.py new file mode 100644 index 000000000..e57f82518 --- /dev/null +++ b/examples/alphaevolve_interop/run_entrypoint.py @@ -0,0 +1,224 @@ +"""Run supported upstream entrypoints locally with explicit configuration swaps.""" + +import argparse +import contextlib +import importlib +import json +import os +import shutil +import sys +import tempfile +import uuid +from pathlib import Path +from types import ModuleType +from unittest.mock import patch + +from openevolve.compat.alpha_evolve import ( + AlphaEvolveClient, + AlphaEvolveExperiment, + local_imports, +) + +EXAMPLES = {"circle_packing": "sum_of_radii", "tsp": "neg_tour_length"} + + +def _load_entrypoint(root, settings): + """Import relative helpers, then unregister them for by-value worker transport.""" + try: + import dotenv + except ImportError as error: + raise ImportError("Install python-dotenv and the upstream example dependencies") from error + + name = "_openevolve_ae_entrypoint_" + uuid.uuid4().hex + package = ModuleType(name) + package.__path__ = [str(root)] + package.__package__ = name + sys.modules[name] = package + try: + # The caller supplies OE settings explicitly. Do not discover/load .env + # files while importing the original entrypoint's configuration globals. + with ( + local_imports(), + patch.object(dotenv, "load_dotenv", return_value=False), + patch.dict(os.environ, settings), + ): + module = importlib.import_module(name + ".src.run_evolution") + return module + finally: + for key in list(sys.modules): + if key == name or key.startswith(name + "."): + del sys.modules[key] + + +def _validate_source(example_dir): + example_dir = Path(example_dir).resolve() + if example_dir.name not in EXAMPLES: + raise ValueError("Supported upstream entrypoints: circle_packing, tsp") + source = example_dir / "src" + required = [source / name for name in ("run_evolution.py", "program.py", "evaluate.py")] + if example_dir.name == "tsp": + required.append(example_dir / "instructions.md") + for path in required: + if not path.is_file(): + raise ValueError("Missing required upstream file: " + str(path)) + if source.is_symlink() or any(path.is_symlink() for path in source.rglob("*")): + raise ValueError("Example src must contain regular files/directories, not symlinks") + instructions = example_dir / "instructions.md" + if instructions.is_symlink(): + raise ValueError("instructions.md must not be a symlink") + return example_dir + + +def run_entrypoint(example_dir, *, api_base, model, api_key=None, iterations=0, output_dir=None): + """Run trusted, byte-identical staged sources; default to seed evaluation only. + + Only the two reviewed entrypoints are supported. Imports are process-global; + use this example runner in a fresh, single-threaded main process. Generation + uses the chosen endpoint and may incur charges when iterations is positive. + Reports operate on the facade's best-only result, not a full search history. + """ + example_dir = _validate_source(example_dir) + if not isinstance(model, str) or not model.strip(): + raise ValueError("model must explicitly name a model served by the OE endpoint") + if api_key is not None and (not isinstance(api_key, str) or not api_key.strip()): + raise ValueError("api_key must be a non-empty string or None") + # Validate options before copying/importing any trusted upstream code. + options = dict( + api_base=api_base, + objective_metric=EXAMPLES[example_dir.name], + api_key=api_key if api_key is not None else "local-placeholder", + iterations=iterations, + ) + AlphaEvolveClient(**options) + destination = Path(output_dir).absolute() if output_dir is not None else None + if destination is not None and destination.resolve().is_relative_to(example_dir): + raise ValueError("output_dir must be outside the upstream example") + + experiments = [] + + def experiment_factory(*args, **kwargs): + experiment = AlphaEvolveExperiment(*args, **kwargs) + experiments.append(experiment) + return experiment + + with contextlib.ExitStack() as stack: + if destination is None: + root = Path( + stack.enter_context(tempfile.TemporaryDirectory(prefix="oe_ae_entrypoint_")) + ) + else: + # Exclusive creation prevents reports or copied sources overwriting + # an existing run. Keep explicit output, including after failure. + destination.mkdir() + root = destination + shutil.copytree( + example_dir / "src", + root / "src", + ignore=shutil.ignore_patterns(".env*", "__pycache__", "*.pyc"), + ) + instructions = example_dir / "instructions.md" + if instructions.is_file(): + shutil.copyfile(instructions, root / "instructions.md") + settings = { + "MODEL": model, + "MODEL_1": model, + "MODEL_2": "", + "MODEL_1_WEIGHT": "1", + "MODEL_2_WEIGHT": "0", + "MAX_PROGRAMS_GENERATED": str(max(1, iterations)), + "MAX_PROGRAMS_EVALUATED": str(max(1, iterations)), + "PARALLEL_EVALUATION": "False", + "CONCURRENCY": "1", + "WORKER_CONCURRENCY": "1", + } + entrypoint = _load_entrypoint(root, settings) + required = { + "main", + "AlphaEvolveClient", + "AlphaEvolveExperiment", + "MAX_PROGRAMS_GENERATED", + "MAX_PROGRAMS_EVALUATED", + "PARALLEL_EVALUATION", + "CONCURRENCY", + } + if example_dir.name == "circle_packing": + required.update( + {"MODEL_1", "MODEL_2", "MODEL_1_WEIGHT", "MODEL_2_WEIGHT", "WORKER_CONCURRENCY"} + ) + else: + required.add("MODEL") + if any(not hasattr(entrypoint, key) for key in required) or not callable(entrypoint.main): + raise ValueError("Upstream entrypoint layout changed; review its configuration first") + + # Override only imported factories and configuration globals. The source + # files and upstream main/evaluator implementations remain unchanged. + def client_factory(*args, **kwargs): + # Explicit OE options take precedence if an upstream constructor + # starts supplying similarly named settings in a later revision. + selected = kwargs | options | {"output_dir": str(root / "oe-output")} + return AlphaEvolveClient(*args, **selected) + + entrypoint.AlphaEvolveClient = client_factory + entrypoint.AlphaEvolveExperiment = experiment_factory + entrypoint.MAX_PROGRAMS_GENERATED = max(1, iterations) + entrypoint.MAX_PROGRAMS_EVALUATED = max(1, iterations) + entrypoint.PARALLEL_EVALUATION = False + entrypoint.CONCURRENCY = 1 + if example_dir.name == "circle_packing": + entrypoint.MODEL_1, entrypoint.MODEL_2 = model, "" + entrypoint.MODEL_1_WEIGHT, entrypoint.MODEL_2_WEIGHT = 1.0, 0.0 + entrypoint.WORKER_CONCURRENCY = 1 + else: + entrypoint.MODEL = model + + entrypoint.main() + if not experiments or any(experiment.state != "COMPLETED" for experiment in experiments): + raise RuntimeError("Upstream entrypoint returned without completing a local experiment") + result = experiments[-1].result + return { + "example": example_dir.name, + "generation_attempt_limit": iterations, + "metrics": result.metrics, + "best_score": result.best_score, + "output_dir": str(destination) if destination is not None else None, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("example_dir", type=Path) + parser.add_argument("--api-base", required=True, help="Explicit OpenEvolve model endpoint") + parser.add_argument("--model", required=True, help="Model served by that endpoint") + parser.add_argument("--api-key-env", help="Read a key from this named environment variable") + parser.add_argument( + "--output-dir", type=Path, help="New directory for sources, OE output and reports" + ) + parser.add_argument( + "--evolve", action="store_true", help="Opt in to model calls, which may incur charges" + ) + parser.add_argument( + "--iterations", type=int, help="Positive generation attempts (default 1 with --evolve)" + ) + args = parser.parse_args(argv) + if args.iterations is not None and (not args.evolve or args.iterations < 1): + parser.error("positive --iterations requires --evolve") + api_key = None + if args.api_key_env is not None: + api_key = os.environ.get(args.api_key_env) + if not api_key or not api_key.strip(): + parser.error("--api-key-env must name a non-empty environment variable") + # Original reports/import-time prints go to stderr; stdout contains the summary. + with contextlib.redirect_stdout(sys.stderr): + result = run_entrypoint( + args.example_dir, + api_base=args.api_base, + model=args.model, + api_key=api_key, + iterations=(args.iterations or 1) if args.evolve else 0, + output_dir=args.output_dir, + ) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_alphaevolve_entrypoint.py b/tests/test_alphaevolve_entrypoint.py new file mode 100644 index 000000000..a6418eee0 --- /dev/null +++ b/tests/test_alphaevolve_entrypoint.py @@ -0,0 +1,402 @@ +"""Entry-point configuration swaps, staged source isolation and offline evolution.""" + +import contextlib +import importlib.util +import io +import json +import os +import secrets +import signal +import sys +import tempfile +import unittest +import warnings +from pathlib import Path +from types import ModuleType +from unittest.mock import patch + +from openevolve.compat.alphaevolve import config_from_alphaevolve + +RUNNER = Path(__file__).resolve().parents[1] / "examples/alphaevolve_interop/run_entrypoint.py" +spec = importlib.util.spec_from_file_location("_local_ae_entrypoint_runner", RUNNER) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class OfflineModel: + def __init__(self, config): + self.model = config.name + + async def generate_with_context(self, system_message, messages, **kwargs): + return "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE" + + +def offline_model(config): + return OfflineModel(config) + + +class TestAlphaEvolveEntrypoint(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + self.addCleanup(lambda: [signal.signal(sig, handler) for sig, handler in handlers.items()]) + self.before = { + name: module for name, module in sys.modules.items() if name.startswith("alpha_evolve") + } + self.options = {"api_base": "http://localhost:8000/v1", "model": "offline-model"} + for name, metric in runner.EXAMPLES.items(): + directory = self.root / name + source = directory / "src" + helpers = source / "utils" + helpers.mkdir(parents=True) + (source / "__init__.py").write_text("") + (helpers / "__init__.py").write_text("") + (directory / "instructions.md").write_text("Maximize x using the original interface.") + (directory / ".env").write_text("DO_NOT_LOAD_EXAMPLE_ENV=1\n") + (source / ".env.private").write_text("DO_NOT_COPY=1\n") + (source / "program.py").write_text("# EVOLVE-BLOCK-START\nx = 1\n# EVOLVE-BLOCK-END\n") + (source / "evaluate.py").write_text( + "from pathlib import Path\n" + "from alpha_evolve.models import (AlphaEvolveEvaluationScore, " + "AlphaEvolveEvaluationScores, AlphaEvolveProgramEvaluation)\n" + "INITIAL_PROGRAM_CODE = Path(__file__).with_name('program.py').read_text()\n" + "def evaluate(candidate):\n" + " namespace = {}\n" + " exec(candidate['content']['files'][0]['content'], namespace)\n" + " return AlphaEvolveProgramEvaluation(scores=AlphaEvolveEvaluationScores(\n" + f" scores=[AlphaEvolveEvaluationScore(metric={metric!r}, score=namespace['x'])]))\n" + ) + (helpers / "report.py").write_text( + "from pathlib import Path\n" + "from alpha_evolve.visualization import get_score\n" + "def report(programs, config):\n" + " root = Path(__file__).resolve().parents[2]\n" + " assert get_score(programs[0], config['metric']) >= 1\n" + " import json\n" + " (root / 'report.json').write_text(json.dumps(config))\n" + ) + (source / "run_evolution.py").write_text( + "import asyncio\n" + "from pathlib import Path\n" + "from dotenv import load_dotenv\n" + "load_dotenv()\n" + "from alpha_evolve.client import AlphaEvolveClient\n" + "from alpha_evolve.experiment import AlphaEvolveExperiment\n" + "from alpha_evolve.controller import run_controller_loop\n" + "from .evaluate import evaluate, INITIAL_PROGRAM_CODE\n" + "from .utils.report import report\n" + "MAX_PROGRAMS_GENERATED = MAX_PROGRAMS_EVALUATED = 100\n" + "MODEL = MODEL_1 = MODEL_2 = 'must-be-replaced'\n" + "MODEL_1_WEIGHT = MODEL_2_WEIGHT = 0.5\n" + "PARALLEL_EVALUATION = True\n" + "CONCURRENCY = WORKER_CONCURRENCY = 32\n" + "def main():\n" + " client = AlphaEvolveClient(project_id='ignored-project')\n" + " experiment = AlphaEvolveExperiment(client, evaluate, " + "MAX_PROGRAMS_EVALUATED, parallel_evaluation=PARALLEL_EVALUATION)\n" + f" if {name!r} == 'circle_packing':\n" + " models = [{'name': MODEL_1, 'weight': MODEL_1_WEIGHT}]\n" + " assert MODEL_2 == '' and MODEL_2_WEIGHT == 0 and WORKER_CONCURRENCY == 1\n" + " else:\n" + " models = [{'name': MODEL}]\n" + " experiment.create_experiment({\n" + " 'program_language': 'python',\n" + " 'problem_description': Path(__file__).resolve().parents[1].joinpath('instructions.md').read_text(),\n" + " 'run_settings': {'max_programs': MAX_PROGRAMS_GENERATED, 'concurrency': CONCURRENCY},\n" + " 'generation_settings': {'models': models},\n" + " })\n" + " experiment.create_initial_program({'content': {'files': [{'path': 'program.py', 'content': INITIAL_PROGRAM_CODE}]}})\n" + " experiment.start_experiment()\n" + " asyncio.run(run_controller_loop(experiment))\n" + " report(experiment.list_programs()['alphaEvolvePrograms'], {\n" + f" 'metric': {metric!r}, 'model': models[0]['name'],\n" + " 'parallel': PARALLEL_EVALUATION, 'concurrency': CONCURRENCY,\n" + " 'budget': MAX_PROGRAMS_GENERATED, 'api_base': client.api_base,\n" + " })\n" + ) + self.example = self.root / "circle_packing" + + def tearDown(self): + self.assertEqual( + { + name: module + for name, module in sys.modules.items() + if name.startswith("alpha_evolve") + }, + self.before, + ) + self.assertFalse(any(name.startswith("_openevolve_ae_entrypoint_") for name in sys.modules)) + + def offline_config(self, *args, **kwargs): + config = config_from_alphaevolve(*args, **kwargs) + config.database.num_islands = 1 + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = offline_model + return config + + def run_offline(self, example, **options): + with ( + patch( + "openevolve.compat.alpha_evolve.experiment.config_from_alphaevolve", + side_effect=self.offline_config, + ), + patch("socket.socket.connect", side_effect=AssertionError("network")), + warnings.catch_warnings(), + ): + warnings.simplefilter("ignore", UserWarning) + return runner.run_entrypoint(example, **(self.options | options)) + + def test_original_main_relative_helpers_and_real_workers_run_with_config_swaps(self): + for name, metric in runner.EXAMPLES.items(): + example = self.root / name + originals = { + path.relative_to(example): path.read_bytes() + for path in example.rglob("*") + if path.is_file() + } + for iterations in (0, 1): + output = self.root / f"output-{name}-{iterations}" + with self.subTest(name=name, iterations=iterations): + result = self.run_offline(example, iterations=iterations, output_dir=output) + self.assertEqual(result["metrics"][metric], 1 + iterations) + self.assertEqual(result["generation_attempt_limit"], iterations) + report = json.loads((output / "report.json").read_text()) + self.assertEqual(report["model"], "offline-model") + self.assertEqual(report["budget"], max(1, iterations)) + self.assertFalse(report["parallel"]) + self.assertEqual(report["concurrency"], 1) + for path, content in originals.items(): + self.assertEqual((example / path).read_bytes(), content) + if path.suffix == ".py": + self.assertEqual((output / path).read_bytes(), content) + self.assertFalse((output / ".env").exists()) + self.assertFalse((output / "src/.env.private").exists()) + self.assertTrue((output / "oe-output/best/best_program.py").is_file()) + + def test_temporary_run_cleans_up_sources_and_report(self): + seen = [] + original_loader = runner._load_entrypoint + + def load(root, settings): + seen.append(root) + return original_loader(root, settings) + + with patch.object(runner, "_load_entrypoint", side_effect=load): + result = self.run_offline(self.example) + self.assertIsNone(result["output_dir"]) + self.assertFalse(seen[0].exists()) + + def test_upstream_constructor_cannot_override_selected_endpoint_budget_or_output(self): + source = self.example / "src/run_evolution.py" + source.write_text( + source.read_text().replace( + "project_id='ignored-project'", + "project_id='ignored-project', api_base='https://must-not-contact.invalid', " + "iterations=100, output_dir='must-not-create'", + ) + ) + output = self.root / "selected-output" + result = self.run_offline(self.example, output_dir=output) + self.assertEqual(result["generation_attempt_limit"], 0) + report = json.loads((output / "report.json").read_text()) + self.assertEqual(report["api_base"], self.options["api_base"]) + self.assertTrue((output / "oe-output/best/best_program.py").is_file()) + + def test_explicit_output_must_be_new_and_outside_source_tree(self): + existing = self.root / "existing" + existing.mkdir() + marker = existing / "keep.txt" + marker.write_text("user data") + for destination, error in ( + (existing, FileExistsError), + (self.example / "new-output", ValueError), + ): + with ( + self.subTest(destination=destination), + patch.object(runner, "_load_entrypoint") as load, + ): + with self.assertRaises(error): + runner.run_entrypoint(self.example, **self.options, output_dir=destination) + load.assert_not_called() + self.assertEqual(marker.read_text(), "user data") + link = self.root / "dangling-link" + link.symlink_to(self.root / "missing-output") + with self.assertRaises(FileExistsError): + runner.run_entrypoint(self.example, **self.options, output_dir=link) + self.assertTrue(link.is_symlink()) + self.assertFalse(link.exists()) + + def test_bad_configuration_fails_before_creating_output_or_importing(self): + output = self.root / "must-not-exist" + for options in ( + {"model": ""}, + {"api_base": ""}, + {"iterations": -1}, + {"iterations": True}, + {"api_key": ""}, + ): + with self.subTest(options=options), patch.object(runner, "_load_entrypoint") as load: + with self.assertRaises(ValueError): + runner.run_entrypoint( + self.example, **(self.options | options), output_dir=output + ) + load.assert_not_called() + self.assertFalse(output.exists()) + + def test_unsupported_layout_missing_sources_and_symlinks_are_rejected(self): + unsupported = self.root / "signal_processing" + unsupported.mkdir() + with self.assertRaisesRegex(ValueError, "Supported upstream"): + runner.run_entrypoint(unsupported, **self.options) + seed = self.example / "src/program.py" + content = seed.read_text() + seed.unlink() + with self.assertRaisesRegex(ValueError, "Missing required"): + runner.run_entrypoint(self.example, **self.options) + elsewhere = self.root / "outside.py" + elsewhere.write_text(content) + seed.symlink_to(elsewhere) + with self.assertRaisesRegex(ValueError, "symlink"): + runner.run_entrypoint(self.example, **self.options) + + def test_import_disables_dotenv_discovery_and_restores_function_on_success_or_failure(self): + import dotenv + + for fail in (False, True): + with ( + self.subTest(fail=fail), + patch.object( + dotenv, "load_dotenv", side_effect=AssertionError("must not search .env") + ) as original, + ): + if fail: + (self.example / "src/run_evolution.py").write_text( + "raise RuntimeError('failed import')" + ) + with self.assertRaisesRegex(RuntimeError, "failed import"): + self.run_offline(self.example) + else: + self.run_offline(self.example) + self.assertIs(dotenv.load_dotenv, original) + original.assert_not_called() + + def test_import_configuration_wins_over_invalid_ambient_settings_and_restores_environment(self): + source = self.example / "src/run_evolution.py" + source.write_text( + "import os\n" + "assert int(os.environ['MAX_PROGRAMS_GENERATED']) == 1\n" + "assert float(os.environ['MODEL_1_WEIGHT']) == 1.0\n" + "assert os.environ['MODEL_1'] == 'offline-model'\n" + source.read_text() + ) + settings = {"MAX_PROGRAMS_GENERATED": "bad", "MODEL_1_WEIGHT": "bad", "MODEL_1": "ambient"} + with patch.dict(os.environ, settings): + self.run_offline(self.example) + self.assertEqual({key: os.environ[key] for key in settings}, settings) + + def test_existing_cloud_sdk_is_preserved_and_temporary_copy_is_cleaned(self): + original = ModuleType("alpha_evolve") + copies = [] + actual = runner._load_entrypoint + + def load(root, settings): + copies.append(root) + return actual(root, settings) + + with ( + patch.dict(sys.modules, {"alpha_evolve": original}), + patch.object(runner, "_load_entrypoint", side_effect=load), + ): + with self.assertRaisesRegex(RuntimeError, "already loaded"): + self.run_offline(self.example) + self.assertIs(sys.modules["alpha_evolve"], original) + self.assertFalse(copies[0].exists()) + + def test_main_return_without_completed_experiment_is_an_error(self): + source = self.example / "src/run_evolution.py" + source.write_text(source.read_text().replace("def main():", "def main():\n return")) + with self.assertRaisesRegex(RuntimeError, "without completing"): + self.run_offline(self.example) + + def test_layout_drift_and_import_interruptions_leave_no_aliases(self): + source = self.example / "src/run_evolution.py" + original = source.read_text() + source.write_text(original.replace("WORKER_CONCURRENCY = 32", "REMOVED_WORKER = 32")) + with self.assertRaisesRegex(ValueError, "layout changed"): + self.run_offline(self.example) + source.write_text("raise KeyboardInterrupt()") + with self.assertRaises(KeyboardInterrupt): + self.run_offline(self.example) + + def test_cli_seed_default_evolution_opt_in_and_key_selection(self): + token = secrets.token_urlsafe(20) + for options, expected in ( + ([], 0), + (["--evolve"], 1), + (["--evolve", "--iterations", "2"], 2), + ): + output = io.StringIO() + with ( + self.subTest(options=options), + patch.dict(os.environ, {"OE_ENTRYPOINT_TEST_KEY": token}), + patch.object( + runner, "run_entrypoint", return_value={"metrics": {"score": 1}} + ) as run, + contextlib.redirect_stdout(output), + ): + runner.main( + [ + str(self.example), + "--api-base", + "local", + "--model", + "model", + "--api-key-env", + "OE_ENTRYPOINT_TEST_KEY", + *options, + ] + ) + self.assertEqual(run.call_args.kwargs["iterations"], expected) + self.assertEqual(run.call_args.kwargs["api_key"], token) + self.assertEqual(json.loads(output.getvalue()), {"metrics": {"score": 1}}) + self.assertNotIn(token, output.getvalue()) + + def test_cli_rejects_invalid_budget_or_key_before_import(self): + for options in ( + ["--iterations", "1"], + ["--evolve", "--iterations", "0"], + ["--api-key-env", "OE_ENTRYPOINT_MISSING"], + ): + with ( + self.subTest(options=options), + patch.object(runner, "run_entrypoint") as run, + contextlib.redirect_stderr(io.StringIO()), + ): + with self.assertRaises(SystemExit): + runner.main( + [str(self.example), "--api-base", "local", "--model", "model", *options] + ) + run.assert_not_called() + + def test_named_key_is_not_replaced_by_ambient_key(self): + with ( + patch.dict(os.environ, {"OE_ENTRYPOINT_BLANK": "", "OPENAI_API_KEY": "ambient"}), + contextlib.redirect_stderr(io.StringIO()), + ): + with self.assertRaises(SystemExit): + runner.main( + [ + str(self.example), + "--api-base", + "local", + "--model", + "model", + "--api-key-env", + "OE_ENTRYPOINT_BLANK", + ] + ) + + +if __name__ == "__main__": + unittest.main()