diff --git a/.github/workflows/alphaevolve-interop.yml b/.github/workflows/alphaevolve-interop.yml new file mode 100644 index 000000000..a9f3fd694 --- /dev/null +++ b/.github/workflows/alphaevolve-interop.yml @@ -0,0 +1,65 @@ +name: AlphaEvolve upstream examples + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/alphaevolve-interop.yml' + - 'openevolve/**' + - 'examples/alphaevolve_interop/**' + - 'tests/alphaevolve_upstream/**' + - 'pyproject.toml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: alphaevolve-interop-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + upstream-examples: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout OpenEvolve + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Read pinned upstream revision + run: | + python - <<'PY' + import os + import re + from pathlib import Path + revision = Path('tests/alphaevolve_upstream/revision.txt').read_text().strip() + if not re.fullmatch(r'[0-9a-f]{40}', revision): + raise SystemExit('Expected a full upstream commit SHA') + with open(os.environ['GITHUB_ENV'], 'a') as environment: + environment.write(f'ALPHAEVOLVE_REVISION={revision}\n') + PY + + - name: Checkout official examples (fixed revision) + uses: actions/checkout@v4 + with: + repository: Google-Cloud-AI/alphaevolve-on-googlecloud + ref: ${{ env.ALPHAEVOLVE_REVISION }} + path: .alphaevolve-upstream + persist-credentials: false + + - name: Install local test dependencies (no Google SDK) + run: | + python -m pip install -e . + python -m pip install -r tests/alphaevolve_upstream/requirements.txt + + - name: Run six original-entrypoint cases and offline tripwire tests + env: + ALPHAEVOLVE_UPSTREAM_DIR: ${{ github.workspace }}/.alphaevolve-upstream + run: python tests/alphaevolve_upstream/test_examples.py -v diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 06092cd93..e62b948a7 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -326,6 +326,61 @@ full SDK replacement, managed-service validation or completion of the issue. python -m unittest discover -s tests -p 'test_alphaevolve_entrypoint.py' -v ``` +### Pinned upstream-example CI + +The separate **AlphaEvolve upstream examples** workflow runs on relevant pull +requests and can be dispatched manually after it is on the default branch. It +uses a standard Ubuntu runner with Python 3.12, read-only repository permissions, +no Secrets, no Google SDK and no paid provider. It has a 15-minute job limit, +cancels superseded runs on the same PR/ref, and uploads no artifacts or caches. +Checkout and dependency installation need Internet access; the example subprocesses +then run with network/SDK-import tripwires and a small allowlist of environment +variables, containing only deliberate dummy credential traps. + +[`tests/alphaevolve_upstream/revision.txt`](../../tests/alphaevolve_upstream/revision.txt) +is the single upstream commit pin. Changing Google's `main` does not silently +change these tests. The suite requires clean example trees at that revision, +with the checkout inside OE's parent workspace (a sibling or nested directory). +Paths are canonicalized and escapes from that workspace are rejected. The +scientific dependencies are test-only and pinned separately; the Matplotlib pin +retains the reviewed upstream plotting API. OE's base dependencies are unchanged. + +There are six original-entrypoint cases: circle packing, TSP and signal +processing, each seed-only and with one fixed comment-only proposal. They use +the existing public runner, unmodified upstream evaluators and real OE workers. +The evolution cases check OE's actual trace for an evaluated child containing +the proposal marker, instead of treating a provider call or successful process +exit alone as proof. The cases also check expected finite seed metrics, exact +original/staged source hashes, alias cleanup and report artifacts. TSP must +produce all five PNG reports and its saved program; signal processing must +retain its evaluator archive under the run directory. + +Python audit hooks record blocked socket/DNS attempts immediately, including in +workers that exit without running `atexit`. Unscoped `alpha_evolve` and Google +imports are blocked and recorded too; the explicit local aliases still work. +Two negative tests check that even caught network/import exceptions remain +observable and fail offline validation. These are tripwires for the trusted +examples, not a security sandbox. Fixed proposals establish compatibility, not +real-model optimization quality or a managed AlphaEvolve service run. + +To reproduce the eight opt-in checks locally from an OE checkout: + +```bash +git clone https://github.com/Google-Cloud-AI/alphaevolve-on-googlecloud.git ../alphaevolve-reference +git -C ../alphaevolve-reference checkout "$(cat tests/alphaevolve_upstream/revision.txt)" +python -m pip install -e . -r tests/alphaevolve_upstream/requirements.txt +ALPHAEVOLVE_UPSTREAM_DIR=../alphaevolve-reference \ + python tests/alphaevolve_upstream/test_examples.py -v +``` + +Run this command in a Python 3.12 environment matching the workflow. No model +server or Google credentials are needed. A missing path, wrong revision, dirty +upstream checkout or offline violation makes explicit execution fail. Temporary +run directories are cleaned up; failures include captured subprocess diagnostics. +This suite is separate from default unit discovery and the existing +`tests/integration` real-model job; neither implicitly downloads the upstream +repository or requires these optional scientific packages. + ## Evaluate an upstream seed locally From an installed OpenEvolve checkout: diff --git a/tests/alphaevolve_upstream/requirements.txt b/tests/alphaevolve_upstream/requirements.txt new file mode 100644 index 000000000..e37750805 --- /dev/null +++ b/tests/alphaevolve_upstream/requirements.txt @@ -0,0 +1,7 @@ +# Optional scientific dependencies for the Python 3.12 upstream-example job. +# No Google SDK, cloud credentials or model server is needed. +numpy==2.5.4 +scipy==1.18.1 +matplotlib==3.10.9 +nest-asyncio==1.6.0 +python-dotenv==1.2.4 diff --git a/tests/alphaevolve_upstream/revision.txt b/tests/alphaevolve_upstream/revision.txt new file mode 100644 index 000000000..5128716fc --- /dev/null +++ b/tests/alphaevolve_upstream/revision.txt @@ -0,0 +1 @@ +34cd919193945c6dc81053f9af594031b54f41a8 diff --git a/tests/alphaevolve_upstream/run_example.py b/tests/alphaevolve_upstream/run_example.py new file mode 100644 index 000000000..ab36bccdc --- /dev/null +++ b/tests/alphaevolve_upstream/run_example.py @@ -0,0 +1,80 @@ +"""Execute one original upstream main through the public CLI, guarded offline.""" + +import argparse +import contextlib +import io +import json +import os +import runpy +import sys +from pathlib import Path +from unittest.mock import patch + + +def main(): + guard = sys.modules.get("sitecustomize") + if not getattr(guard, "ACTIVE", False): + raise RuntimeError("Run via test_examples.py with offline tripwires enabled") + + from offline_model import fixed_model + + from openevolve.compat.alphaevolve import config_from_alphaevolve + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("example", type=Path) + parser.add_argument("iterations", type=int, choices=(0, 1)) + parser.add_argument("workspace", type=Path) + args = parser.parse_args() + runner = Path(__file__).resolve().parents[2] / "examples/alphaevolve_interop/run_entrypoint.py" + argv = [ + str(runner), + str(args.example), + "--api-base", + "http://127.0.0.1:1/v1", + "--model", + "fixed-offline-model", + "--output-dir", + str(args.workspace / "run"), + ] + if args.iterations: + argv += ["--evolve", "--iterations", "1"] + + def configure(*values, **options): + config = config_from_alphaevolve(*values, **options) + if config.llm.api_key != "local-placeholder": + raise AssertionError("Runner must not use the dummy ambient key") + config.database.num_islands = 1 + config.log_level = "WARNING" + config.evolution_trace.enabled = True + config.evolution_trace.include_code = True + config.evolution_trace.include_prompts = False + config.evolution_trace.buffer_size = 1 + config.evolution_trace.output_path = str(args.workspace / "evolution.jsonl") + # Seed-only runs construct the real provider client but make no requests. + # Evolution uses real OE workers with only proposal generation replaced. + if args.iterations: + for model in config.llm.models + config.llm.evaluator_models: + model.init_client = fixed_model + return config + + stdout = io.StringIO() + with ( + patch.object(sys, "argv", argv), + patch( + "openevolve.compat.alpha_evolve.experiment.config_from_alphaevolve", + side_effect=configure, + ), + contextlib.redirect_stdout(stdout), + ): + runpy.run_path(str(runner), run_name="__main__") + result = json.loads(stdout.getvalue()) + if any(name == "alpha_evolve" or name.startswith("alpha_evolve.") for name in sys.modules): + raise AssertionError("SDK aliases leaked after main") + if any(name.startswith("_openevolve_ae_entrypoint_") for name in sys.modules): + raise AssertionError("Temporary upstream package leaked after main") + (args.workspace / "result.json").write_text(json.dumps(result), encoding="utf-8") + (args.workspace / "process.json").write_text(json.dumps({"pid": os.getpid()}), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tests/alphaevolve_upstream/support/offline_model.py b/tests/alphaevolve_upstream/support/offline_model.py new file mode 100644 index 000000000..15df96e9b --- /dev/null +++ b/tests/alphaevolve_upstream/support/offline_model.py @@ -0,0 +1,28 @@ +"""Importable fixed-proposal provider for real OE multiprocessing workers.""" + +import json +import os +import sys +from pathlib import Path + +MARKER = "# AlphaEvolve upstream CI: evaluated fixed proposal." + + +class FixedProposalModel: + def __init__(self, config): + self.model = config.name + + async def generate_with_context(self, system_message, messages, **kwargs): + guard = sys.modules.get("sitecustomize") + if not getattr(guard, "ACTIVE", False): + raise AssertionError("Offline tripwires must also be active in OE workers") + with Path(os.environ["OE_INTEROP_CALL_LOG"]).open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"model": self.model, "pid": os.getpid()}) + "\n") + return ( + "<<<<<<< SEARCH\n# EVOLVE-BLOCK-START\n=======\n" + f"# EVOLVE-BLOCK-START\n{MARKER}\n>>>>>>> REPLACE" + ) + + +def fixed_model(config): + return FixedProposalModel(config) diff --git a/tests/alphaevolve_upstream/support/sitecustomize.py b/tests/alphaevolve_upstream/support/sitecustomize.py new file mode 100644 index 000000000..93533cd34 --- /dev/null +++ b/tests/alphaevolve_upstream/support/sitecustomize.py @@ -0,0 +1,41 @@ +"""Block and durably record network/SDK attempts in dedicated test processes. + +Loaded at interpreter startup via the test driver's explicit PYTHONPATH. These +are test tripwires for trusted examples, not a sandbox for untrusted code. +""" + +import importlib.abc +import json +import os +import sys +from pathlib import Path + +ACTIVE = False + + +def record(kind): + # Write attempts immediately: multiprocessing workers may exit via os._exit, + # and upstream code can catch the exception raised by a tripwire. + path = Path(os.environ["OE_INTEROP_AUDIT_DIR"]) / f"{os.getpid()}.jsonl" + with path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"kind": kind, "pid": os.getpid()}) + "\n") + + +def audit(event, args): + if event in {"socket.connect", "socket.getaddrinfo", "socket.gethostbyname"}: + record(event) + raise AssertionError("Upstream-example test blocked network access") + + +class NoCloudImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".", 1)[0] in {"alpha_evolve", "google"}: + record("cloud_import") + raise AssertionError("Upstream-example test blocked a cloud SDK/auth import") + + +if os.environ.get("OE_INTEROP_AUDIT_DIR"): + sys.addaudithook(audit) + sys.meta_path.insert(0, NoCloudImports()) + record("active") + ACTIVE = True diff --git a/tests/alphaevolve_upstream/test_examples.py b/tests/alphaevolve_upstream/test_examples.py new file mode 100644 index 000000000..b059c31c9 --- /dev/null +++ b/tests/alphaevolve_upstream/test_examples.py @@ -0,0 +1,247 @@ +"""Opt-in integration checks against pinned, unchanged official AE examples.""" + +import hashlib +import json +import math +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SUITE = Path(__file__).resolve().parent +REPO = SUITE.parents[1] +MARKER = "# AlphaEvolve upstream CI: evaluated fixed proposal." +EXAMPLES = { + "circle_packing": ("sum_of_radii", 0.9414554065860808), + "tsp": ("neg_tour_length", -7.223903565915199), + "signal_processing": ("overall_score", 0.49904861783269006), +} + + +def source_hashes(example): + paths = list((example / "src").rglob("*.py")) + if (example / "instructions.md").is_file(): + paths.append(example / "instructions.md") + return { + str(path.relative_to(example)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in paths + } + + +def json_lines(path): + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +class OfflineSubprocessTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="oe_upstream_test_") + self.addCleanup(temporary.cleanup) + self.workspace = Path(temporary.name) + self.audit_dir = self.workspace / "audit" + self.audit_dir.mkdir() + + def run_python(self, arguments): + # Do not forward developer/CI secrets, proxies, SDK paths or Python startup + # settings. The only credential-like values are deliberate dummy traps. + environment = { + key: os.environ[key] + for key in ("PATH", "HOME", "TMPDIR", "SYSTEMROOT", "WINDIR") + if key in os.environ + } + environment.update( + PYTHONPATH=os.pathsep.join((str(SUITE / "support"), str(REPO))), + PYTHONNOUSERSITE="1", + MPLBACKEND="Agg", + MPLCONFIGDIR=str(self.workspace / "matplotlib"), + OMP_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + OE_INTEROP_AUDIT_DIR=str(self.audit_dir), + OE_INTEROP_CALL_LOG=str(self.workspace / "calls.jsonl"), + OPENAI_API_KEY="dummy-ambient-key-must-not-be-used", + GOOGLE_APPLICATION_CREDENTIALS=str(self.workspace / "nonexistent-credentials.json"), + SAVE_PROGRAM_DIR=str(self.workspace / "must-not-write-here"), + ) + return subprocess.run( + [sys.executable, *arguments], + cwd=self.workspace, + env=environment, + capture_output=True, + text=True, + timeout=120, + ) + + def audits(self): + return [event for path in self.audit_dir.glob("*.jsonl") for event in json_lines(path)] + + def check_offline(self): + events = self.audits() + self.assertTrue(any(event["kind"] == "active" for event in events), "Tripwires not loaded") + self.assertEqual([event for event in events if event["kind"] != "active"], []) + + +class TestOfflineTripwires(OfflineSubprocessTest): + def test_caught_network_attempts_still_fail_offline_validation(self): + result = self.run_python( + [ + "-c", + "import socket\ntry: socket.create_connection(('127.0.0.1', 1))\nexcept AssertionError: pass", + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + with self.assertRaises(AssertionError): + self.check_offline() + self.assertTrue(any(event["kind"].startswith("socket.") for event in self.audits())) + + def test_caught_cloud_sdk_or_auth_imports_still_fail_offline_validation(self): + result = self.run_python( + [ + "-c", + "import importlib\nfor name in ('alpha_evolve', 'google.auth'):\n" + " try: importlib.import_module(name)\n except AssertionError: pass", + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + with self.assertRaises(AssertionError): + self.check_offline() + self.assertEqual(sum(event["kind"] == "cloud_import" for event in self.audits()), 2) + + +class TestUpstreamExamples(OfflineSubprocessTest): + @classmethod + def setUpClass(cls): + directory = os.environ.get("ALPHAEVOLVE_UPSTREAM_DIR") + if not directory: + raise unittest.SkipTest("Set ALPHAEVOLVE_UPSTREAM_DIR to opt in to upstream examples") + cls.upstream = Path(os.path.realpath(directory)) + if not cls.upstream.is_relative_to(REPO.parent): + raise RuntimeError("Keep the upstream checkout inside the OE parent workspace") + revision = (SUITE / "revision.txt").read_text().strip() + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise RuntimeError("revision.txt must contain a full upstream commit SHA") + actual = subprocess.check_output( + ["git", "-C", str(cls.upstream), "rev-parse", "HEAD"], text=True + ).strip() + if actual != revision: + raise RuntimeError(f"Expected pinned upstream revision {revision}, got {actual}") + paths = [f"examples/{name}" for name in EXAMPLES] + dirty = subprocess.check_output( + [ + "git", + "-C", + str(cls.upstream), + "status", + "--porcelain", + "--untracked-files=all", + "--", + *paths, + ], + text=True, + ) + if dirty: + raise RuntimeError("Use a clean checkout of the pinned upstream examples") + + def run_case(self, name, iterations): + example = self.upstream / "examples" / name + original = source_hashes(example) + self.assertIn("src/program.py", original) + self.assertIn("src/evaluate.py", original) + result = self.run_python( + [str(SUITE / "run_example.py"), str(example), str(iterations), str(self.workspace)] + ) + self.assertEqual(result.returncode, 0, result.stderr[-8000:] + result.stdout[-2000:]) + self.check_offline() + self.assertEqual(source_hashes(example), original) + output = self.workspace / "run" + self.assertEqual(source_hashes(output), original) + self.assertFalse((output / ".env").exists()) + self.assertFalse((self.workspace / "must-not-write-here").exists()) + summary = json.loads((self.workspace / "result.json").read_text()) + metric, baseline = EXAMPLES[name] + self.assertEqual(summary["example"], name) + self.assertEqual(summary["generation_attempt_limit"], iterations) + self.assertNotIn("error", summary["metrics"]) + score = summary["metrics"][metric] + self.assertTrue(math.isfinite(score) and math.isclose(score, baseline, rel_tol=1e-6)) + self.assertEqual(summary["best_score"], score) + self.assertTrue((output / "oe-output/best/best_program.py").is_file()) + calls = json_lines(self.workspace / "calls.jsonl") + traces = json_lines(self.workspace / "evolution.jsonl") + self.assertEqual(len(calls), iterations) + self.assertEqual(len(traces), iterations) + if iterations: + self.assertEqual(calls[0]["model"], "fixed-offline-model") + process = json.loads((self.workspace / "process.json").read_text()) + self.assertNotEqual( + calls[0]["pid"], process["pid"], "Proposal must run in a real OE worker" + ) + self.assertIn(MARKER, traces[0]["child_code"]) + self.assertEqual(traces[0]["parent_code"], (example / "src/program.py").read_text()) + self.assertNotIn("error", traces[0]["child_metrics"]) + self.assertTrue( + math.isclose(traces[0]["child_metrics"][metric], baseline, rel_tol=1e-6) + ) + if name == "tsp": + images = {path.name for path in (output / "report").glob("*.png")} + self.assertEqual( + images, + { + "best_all_instances.png", + "evolution_progress.png", + "per_instance_comparison.png", + "seed_vs_best.png", + "tour_length_distribution.png", + }, + ) + for image in images: + self.assertEqual((output / "report" / image).read_bytes()[:8], b"\x89PNG\r\n\x1a\n") + self.assertTrue((output / "evolved_program/main.py").is_file()) + elif name == "circle_packing": + self.assertIn("Visualizing Rank 1:", result.stderr) + self.assertNotIn("ERROR visualizing program", result.stderr) + self.assertNotIn("ERROR: construct_packing", result.stderr) + else: + self.assertEqual(summary["metrics"]["success_rate"], 1.0) + archives = list((output / "programs").rglob("*.json")) + self.assertEqual(len(archives), 1) + archived = json.loads(archives[0].read_text()) + self.assertEqual(MARKER in archived["content"]["files"][0]["content"], bool(iterations)) + print( + json.dumps( + { + "example": name, + "iterations": iterations, + "score": score, + "evaluated_proposals": len(traces), + "network_or_cloud_attempts": 0, + } + ), + flush=True, + ) + + def test_circle_packing_seed(self): + self.run_case("circle_packing", 0) + + def test_circle_packing_proposal(self): + self.run_case("circle_packing", 1) + + def test_tsp_seed(self): + self.run_case("tsp", 0) + + def test_tsp_proposal(self): + self.run_case("tsp", 1) + + def test_signal_processing_seed(self): + self.run_case("signal_processing", 0) + + def test_signal_processing_proposal(self): + self.run_case("signal_processing", 1) + + +if __name__ == "__main__": + # Explicit CI/manual execution must never go green by skipping all examples. + if not os.environ.get("ALPHAEVOLVE_UPSTREAM_DIR"): + raise SystemExit("ALPHAEVOLVE_UPSTREAM_DIR is required; see the interop walkthrough") + unittest.main()