From be7f32646d141e42c6c29ab4bd6b11e20a973181 Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Fri, 23 Oct 2026 18:53:57 -0700 Subject: [PATCH] test(compat): verify OpenEvolve bundle round trips --- .github/workflows/alphaevolve-interop.yml | 5 + examples/alphaevolve_interop/README.md | 52 ++++++ tests/alphaevolve_upstream/run_roundtrip.py | 184 +++++++++++++++++++ tests/alphaevolve_upstream/test_roundtrip.py | 118 ++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 tests/alphaevolve_upstream/run_roundtrip.py create mode 100644 tests/alphaevolve_upstream/test_roundtrip.py diff --git a/.github/workflows/alphaevolve-interop.yml b/.github/workflows/alphaevolve-interop.yml index a9f3fd694..cfe9a433f 100644 --- a/.github/workflows/alphaevolve-interop.yml +++ b/.github/workflows/alphaevolve-interop.yml @@ -7,6 +7,8 @@ on: - '.github/workflows/alphaevolve-interop.yml' - 'openevolve/**' - 'examples/alphaevolve_interop/**' + - 'examples/function_minimization/**' + - 'examples/circle_packing/**' - 'tests/alphaevolve_upstream/**' - 'pyproject.toml' workflow_dispatch: @@ -63,3 +65,6 @@ jobs: env: ALPHAEVOLVE_UPSTREAM_DIR: ${{ github.workspace }}/.alphaevolve-upstream run: python tests/alphaevolve_upstream/test_examples.py -v + + - name: Round-trip two OpenEvolve examples through relocated export bundles + run: python tests/alphaevolve_upstream/test_roundtrip.py -v diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index e62b948a7..27ed355f9 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -709,6 +709,58 @@ python -m unittest discover -s tests -p 'test_alphaevolve_bundle.py' -v python -m unittest discover -s tests -p 'test_alphaevolve_launch.py' -v ``` +### Verify a complete local round trip + +The dedicated interop workflow also exercises two existing OpenEvolve examples +through this complete path, without Google's SDK or a model service: + +```text +OE Python seed + evaluator + YAML + -> export_to_alphaevolve + -> relocate bundle and remove the original input directory + -> config_from_alphaevolve(experiment.json) + -> generated candidate evaluator -> adapt_evaluator + -> real OpenEvolve seed evaluation + one evaluated proposal +``` + +From an installed source checkout, reproduce just these checks with: + +```bash +python tests/alphaevolve_upstream/test_roundtrip.py -v +``` + +Unlike the original-entrypoint tests, this command needs no AlphaEvolve source +checkout. It uses the unchanged `function_minimization` and `circle_packing` +sources from this repository. The helper runs from an unrelated temporary +working directory, checks source hashes and configuration mappings, and removes +temporary import registrations before transporting evaluators to OE workers. +Both the seed and a fixed comment-only proposal are evaluated on each side. +The tests compare all numeric seed/child metrics except elapsed evaluation time, +check positive finite fitness, and require worker evaluation records and actual +evolution traces. A provider call or successful process exit alone is insufficient. +The existing test-only startup tripwires block and record network and cloud SDK +attempts in the helper, workers and evaluator subprocesses. These are safeguards +for trusted examples, not a security sandbox. + +This is execution equivalence of the **supported subset under controlled inputs**, +not a lossless search/configuration round trip. The test bounds the budget to one, +fixes NumPy's random stream per evaluation, and explicitly aligns one island, +single-stage single-worker evaluation and diff mode. The circle example needs +an explicit model-name mapping; model weights and the mapped names are checked +before replacing generation with the fixed offline provider. The imported prompt +has the documented `Problem description:` prefix. Export warnings for omitted +settings are checked; original artifacts and timing are not expected to survive. +There is no claim of real-model improvement or of preservation of the examples' +original multi-island/cascade behavior. + +For #468, the three original local entrypoints and this OE-to-bundle-to-OE path +now have automated offline coverage. **An authenticated managed AlphaEvolve run +remains a separate acceptance check.** Neither these tests nor GitHub Actions +exercise Google's service, verify project access/live model availability, or +establish cloud cost limits. A maintainer with service access can review a bundle, +run its local `check.py` and preflight, then explicitly test `--run` as documented +above before deciding whether to close #468 or track remaining scope separately. + ## Import a local Python experiment configuration `config_from_alphaevolve` imports an explicit experiment dictionary without diff --git a/tests/alphaevolve_upstream/run_roundtrip.py b/tests/alphaevolve_upstream/run_roundtrip.py new file mode 100644 index 000000000..cdc7a734e --- /dev/null +++ b/tests/alphaevolve_upstream/run_roundtrip.py @@ -0,0 +1,184 @@ +"""Execute an OE example before and after bundle export/import, guarded offline.""" + +import argparse +import copy +import hashlib +import json +import os +import shutil +import sys +import warnings +from pathlib import Path + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + guard = sys.modules.get("sitecustomize") + if not getattr(guard, "ACTIVE", False): + raise RuntimeError("Run via test_roundtrip.py with offline tripwires enabled") + + import numpy as np + from offline_model import fixed_model + + from openevolve.api import run_evolution + from openevolve.compat.alpha_evolve import load_evaluator + from openevolve.compat.alphaevolve import adapt_evaluator, config_from_alphaevolve + from openevolve.compat.alphaevolve_bundle import export_to_alphaevolve + from openevolve.compat.alphaevolve_launch import _bundle_evaluator + from openevolve.config import Config + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("example", choices=("function_minimization", "circle_packing")) + parser.add_argument("workspace", type=Path) + args = parser.parse_args() + repo = Path(__file__).resolve().parents[2] + example = repo / "examples" / args.example + config_name = ( + "config.yaml" if args.example == "function_minimization" else "config_phase_1.yaml" + ) + inputs = args.workspace / "inputs" + inputs.mkdir() + for name in ("initial_program.py", "evaluator.py", config_name): + shutil.copyfile(example / name, inputs / name) + original_hashes = { + name: digest(inputs / name) for name in ("initial_program.py", "evaluator.py") + } + source = Config.from_yaml(inputs / config_name) + source.language = "python" + source.max_iterations = 1 # Bound this test; do not run the example's full budget. + before = copy.deepcopy(source) + mapping = ( + { + "google/gemini-2.0-flash-001": "gemini-2.5-flash", + "anthropic/claude-3.7-sonnet": "gemini-2.5-pro", + } + if args.example == "circle_packing" + else None + ) + output = args.workspace / "bundle" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + export_to_alphaevolve( + inputs / "initial_program.py", + inputs / "evaluator.py", + source, + output, + title=args.example, + problem_description=source.prompt.system_message, + objective_metric="combined_score", + model_mapping=mapping, + ) + assert source == before, "Export must not mutate the source configuration" + export_warnings = [str(item.message) for item in caught] + relocated = args.workspace / "relocated" + output.rename(relocated) + experiment = json.loads((relocated / "experiment.json").read_text()) + settings = json.loads((relocated / "evaluation.json").read_text()) + imported = config_from_alphaevolve( + experiment, api_base="http://127.0.0.1:1/v1", api_key="roundtrip-placeholder" + ) + assert imported.max_iterations == source.max_iterations == 1 + assert imported.language == source.language == "python" + expected_models = [ + ((mapping or {}).get(model.name, model.name), model.weight) for model in source.llm.models + ] + assert [(model.name, model.weight) for model in imported.llm.models] == expected_models + assert imported.prompt.system_message == "Problem description:\n" + source.prompt.system_message + assert not imported.evaluator.cascade_evaluation + assert imported.evaluator.parallel_evaluations == 1 + for model in imported.llm.models + imported.llm.evaluator_models: + assert model.api_base == "http://127.0.0.1:1/v1" + assert model.api_key == "roundtrip-placeholder" + + # Loading the generated entrypoint exercises its copied evaluator and settings. + # Remove temporary module registrations BEFORE serialization to real workers. + direct_evaluate = load_evaluator(inputs / "evaluator.py", "evaluate") + with _bundle_evaluator(relocated) as candidate_evaluate: + returned_evaluate = adapt_evaluator( + candidate_evaluate, + **{ + "objective_metric": settings["objective_metric"], + "failure_sentinel": settings["failure_score"], + "failure_score": settings["failure_score"], + }, + ) + assert not any(name.startswith("_openevolve_alphaevolve_") for name in sys.modules) + assert not any(name.startswith("_openevolve_local_ae_evaluator_") for name in sys.modules) + # Neither run may depend on the original input directory or working directory. + seed = (relocated / "src/program.py").read_text() + shutil.rmtree(inputs) + baseline = copy.deepcopy(source) + for model, (name, _) in zip(baseline.llm.models, expected_models): + model.name = name # The circle example explicitly remaps two provider models. + + results = {} + for label, config, evaluator in ( + ("direct", baseline, direct_evaluate), + ("roundtrip", imported, returned_evaluate), + ): + # Deliberately align documented losses for a one-proposal comparison. + # This is NOT equivalence of the original multi-island/cascade search. + config.database.num_islands = 1 + config.evaluator.cascade_evaluation = False + config.evaluator.parallel_evaluations = 1 + config.evaluator.use_llm_feedback = False + config.diff_based_evolution = True + config.log_level = "WARNING" + config.evolution_trace.enabled = True + config.evolution_trace.include_code = True + config.evolution_trace.include_prompts = False + config.evolution_trace.buffer_size = 1 + config.evolution_trace.output_path = str(args.workspace / (label + ".jsonl")) + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = fixed_model + + def evaluate(path): + # The function-minimization seed uses global NumPy RNG. Fix its input + # stream per evaluation on BOTH paths, without editing either source. + np.random.seed(123) + worker_guard = sys.modules.get("sitecustomize") + if not getattr(worker_guard, "ACTIVE", False): + raise AssertionError("Offline tripwires must be active during evaluation") + result = evaluator(path) + with (args.workspace / (label + "-evaluations.jsonl")).open("a") as stream: + stream.write(json.dumps({"pid": os.getpid(), "hash": digest(Path(path))}) + "\n") + return result + + result = run_evolution( + seed if label == "direct" else relocated / "src/program.py", + evaluate, + config=config, + output_dir=str(args.workspace / (label + "-output")), + ) + results[label] = { + "best_score": result.best_score, + "metrics": result.metrics, + "code": result.best_code, + } + copied_hashes = { + "initial_program.py": digest(relocated / "src/program.py"), + "evaluator.py": digest(relocated / "src/openevolve_evaluator.py"), + } + assert copied_hashes == original_hashes + assert not inputs.exists() and not output.exists() + (args.workspace / "roundtrip-result.json").write_text( + json.dumps( + { + "example": args.example, + "pid": os.getpid(), + "results": results, + "models": expected_models, + "export_warnings": export_warnings, + "sources_unchanged": True, + "inputs_removed": True, + } + ), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/tests/alphaevolve_upstream/test_roundtrip.py b/tests/alphaevolve_upstream/test_roundtrip.py new file mode 100644 index 000000000..0563a5ff9 --- /dev/null +++ b/tests/alphaevolve_upstream/test_roundtrip.py @@ -0,0 +1,118 @@ +"""Offline execution equivalence of OE examples through exported AE bundles.""" + +import hashlib +import json +import math +import sys +import unittest + +from test_examples import REPO, SUITE, OfflineSubprocessTest, json_lines + +MARKER = "# AlphaEvolve upstream CI: evaluated fixed proposal." + + +class TestBundleRoundTrip(OfflineSubprocessTest): + def run_case(self, name): + example = REPO / "examples" / name + originals = { + file: hashlib.sha256((example / file).read_bytes()).hexdigest() + for file in ("initial_program.py", "evaluator.py") + } + completed = self.run_python([str(SUITE / "run_roundtrip.py"), name, str(self.workspace)]) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.check_offline() + summary = json.loads((self.workspace / "roundtrip-result.json").read_text()) + self.assertTrue(summary["sources_unchanged"] and summary["inputs_removed"]) + self.assertEqual( + originals, + {file: hashlib.sha256((example / file).read_bytes()).hexdigest() for file in originals}, + ) + self.assertFalse((self.workspace / "inputs").exists()) + self.assertFalse((self.workspace / "bundle").exists()) + warnings = "\n".join(summary["export_warnings"]) + for field in ("database.num_islands", "evaluator.cascade_evaluation", "llm.api_base"): + self.assertIn(field, warnings) + if name == "circle_packing": + self.assertIn("Model names explicitly remapped", warnings) + + seed = (example / "initial_program.py").read_text() + expected_child = seed.replace( + "# EVOLVE-BLOCK-START\n", f"# EVOLVE-BLOCK-START\n{MARKER}\n", 1 + ) + expected_hashes = { + hashlib.sha256(code.encode()).hexdigest() for code in (seed, expected_child) + } + traces = {} + for label in ("direct", "roundtrip"): + result = summary["results"][label] + metrics = result["metrics"] + self.assertNotIn("error", metrics) + self.assertTrue(all(math.isfinite(value) for value in metrics.values())) + self.assertGreater(result["best_score"], 0) + self.assertEqual(result["best_score"], metrics["combined_score"]) + self.assertIn(result["code"], (seed, expected_child)) + traces[label] = json_lines(self.workspace / (label + ".jsonl")) + self.assertEqual(len(traces[label]), 1, "Must evaluate one actual proposal per run") + trace = traces[label][0] + self.assertEqual(trace["parent_code"], seed) + self.assertEqual(trace["child_code"], expected_child) + self.assertNotIn("error", trace["child_metrics"]) + self.assertGreater(trace["child_metrics"]["combined_score"], 0) + evaluations = json_lines(self.workspace / (label + "-evaluations.jsonl")) + self.assertEqual(len(evaluations), 2, "Both seed and child must be evaluated") + self.assertEqual({item["hash"] for item in evaluations}, expected_hashes) + self.assertTrue(any(item["pid"] != summary["pid"] for item in evaluations)) + self.assertTrue((self.workspace / (label + "-output/best/best_program.py")).is_file()) + + # Runtime is inherently variable. All other returned numeric metrics must + # survive the evaluator-envelope round trip, for seed AND evaluated child. + for left, right in ( + (summary["results"]["direct"]["metrics"], summary["results"]["roundtrip"]["metrics"]), + (traces["direct"][0]["parent_metrics"], traces["roundtrip"][0]["parent_metrics"]), + (traces["direct"][0]["child_metrics"], traces["roundtrip"][0]["child_metrics"]), + ): + self.assertEqual(set(left), set(right)) + for metric in left: + if metric != "eval_time": + self.assertTrue( + math.isclose(left[metric], right[metric], rel_tol=1e-9, abs_tol=1e-12), + metric, + ) + calls = json_lines(self.workspace / "calls.jsonl") + self.assertEqual(len(calls), 2) + self.assertTrue(all(item["pid"] != summary["pid"] for item in calls)) + self.assertTrue(all(item["model"] in dict(summary["models"]) for item in calls)) + print( + json.dumps( + { + "example": name, + "direct_score": summary["results"]["direct"]["best_score"], + "roundtrip_score": summary["results"]["roundtrip"]["best_score"], + "evaluated_proposals": len(calls), + "network_or_cloud_attempts": 0, + } + ), + flush=True, + ) + + def test_function_minimization_roundtrip(self): + self.run_case("function_minimization") + + def test_circle_packing_roundtrip(self): + self.run_case("circle_packing") + + def test_unguarded_helper_fails_before_executing_examples(self): + completed = self.run_python( + [ + "-c", + "import runpy, sitecustomize; sitecustomize.ACTIVE = False; " + f"runpy.run_path({str(SUITE / 'run_roundtrip.py')!r}, run_name='__main__')", + ] + ) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("offline tripwires", completed.stderr) + self.assertEqual(list(self.workspace.glob("*-output")), []) + + +if __name__ == "__main__": + unittest.main()