diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d0ebefd0b3..6c3213c4c3 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -760,26 +760,14 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras - - name: Run dense all-rules and delta edit-loop guards under 4 GiB + # Each timed compile runs in its own 4 GiB transient scope; pytest stays outside it. + - name: Run dense all-rules and delta edit-loop guards with each compile under 4 GiB env: BENCHMARK_MODELS: ${{ matrix.models }} - MEMORY_LIMIT_BYTES: "4294967296" + SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES: "4294967296" run: | - sudo systemd-run \ - --unit="sqlbuild-dense-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \ - --uid="$(id -u)" \ - --gid="$(id -g)" \ - --same-dir \ - --setenv="HOME=${HOME}" \ - --setenv="PATH=${PATH}" \ - --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \ - --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \ - --property="MemorySwapMax=0" \ - --wait \ - --collect \ - --pipe \ - make test-e2e-dense-compile-performance \ - SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" + make test-e2e-dense-compile-performance \ + SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" - name: Run diagnostic-heavy compile guards if: matrix.models == 1000 diff --git a/Makefile b/Makefile index fc29dc0dd1..a2c28cbad3 100644 --- a/Makefile +++ b/Makefile @@ -334,7 +334,7 @@ test-e2e-dense-compile-performance: -n auto --dist loadfile \ -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \ -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes -# DuckShop 10K runs after the dense guards, alone, so it never shares the memory limit with them. +# DuckShop 10K runs after the dense guards, alone, so its compile never overlaps theirs. if [ "$(SQLBUILD_BENCHMARK_MODELS)" = "10000" ]; then \ env PYTHONUNBUFFERED=1 uv run pytest benchmarks/duckshop-10k/tests -q --color=yes && \ env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ diff --git a/fensu.toml b/fensu.toml index 54ddd88f0a..ed50609ba3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -992,11 +992,21 @@ modules = ["scripts.cold_compile_performance.main.assert_required_cgroup_memory_ symbols = ["assert_required_cgroup_memory_limit"] reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.compile_memory_scope_command"] +symbols = ["compile_memory_scope_command"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + [[targets.python.dead_code.roots]] modules = ["scripts.cold_compile_performance.main.read_compile_measurement"] symbols = ["read_compile_measurement"] reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.require_compile_memory_scope"] +symbols = ["require_compile_memory_scope"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + [[targets.python.dead_code.roots]] modules = ["scripts.cold_compile_performance.main.semantic_compile_fingerprint"] symbols = ["semantic_compile_fingerprint"] diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 7ed6e83829..d964f21891 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -1,6 +1,7 @@ """Constants for fresh-process compile performance guards.""" UNLIMITED_CGROUP_MEMORY_VALUE: str = "max" +COMPILE_MEMORY_LIMIT_ENV_VAR: str = "SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES" COMPILE_MEASUREMENT_FIELD_COUNT: int = 6 DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99) diff --git a/scripts/cold_compile_performance/main/compile_memory_scope_command.py b/scripts/cold_compile_performance/main/compile_memory_scope_command.py new file mode 100644 index 0000000000..746b9b1c36 --- /dev/null +++ b/scripts/cold_compile_performance/main/compile_memory_scope_command.py @@ -0,0 +1,42 @@ +"""Wrap one timed compile in a memory-limited transient systemd scope.""" + +import os +import shutil +import uuid + +from scripts.cold_compile_performance.constants import COMPILE_MEMORY_LIMIT_ENV_VAR +from scripts.cold_compile_performance.exceptions import CgroupMemoryLimitError + + +def compile_memory_scope_command(*, label: str) -> tuple[str, ...]: + """Prefix for a scope with the configured per-compile limit; refuse to run without it.""" + + raw_limit: str | None = os.environ.get(COMPILE_MEMORY_LIMIT_ENV_VAR) + if raw_limit is None: + return () + unit: str = f"sqlbuild-compile-{label}-{os.getpid()}-{uuid.uuid4().hex[:8]}" + systemd_run: str | None = shutil.which("systemd-run") + sudo: str | None = shutil.which("sudo") + caller_path_over_sudo_secure_path: str = f"PATH={os.environ.get('PATH', '')}" + if systemd_run is None or sudo is None: + raise CgroupMemoryLimitError( + f"{COMPILE_MEMORY_LIMIT_ENV_VAR}={raw_limit} requires systemd-run and sudo to start " + f"the transient scope {unit}.scope; refusing to compile without the memory limit" + ) + return ( + sudo, + "--non-interactive", + "--preserve-env", + systemd_run, + "--scope", + "--quiet", + "--collect", + f"--unit={unit}", + f"--uid={os.getuid()}", + f"--gid={os.getgid()}", + f"--property=MemoryMax={int(raw_limit)}", + "--property=MemorySwapMax=0", + "--", + "/usr/bin/env", + caller_path_over_sudo_secure_path, + ) diff --git a/scripts/cold_compile_performance/main/require_compile_memory_scope.py b/scripts/cold_compile_performance/main/require_compile_memory_scope.py new file mode 100644 index 0000000000..8c188e8e17 --- /dev/null +++ b/scripts/cold_compile_performance/main/require_compile_memory_scope.py @@ -0,0 +1,16 @@ +"""Require a per-compile memory limit when guards run in CI.""" + +import os + +from scripts.cold_compile_performance.constants import COMPILE_MEMORY_LIMIT_ENV_VAR +from scripts.cold_compile_performance.exceptions import CgroupMemoryLimitError + + +def require_compile_memory_scope() -> None: + """Refuse to run in CI unless each timed compile gets its own memory-limited scope.""" + + if os.environ.get("CI") and COMPILE_MEMORY_LIMIT_ENV_VAR not in os.environ: + raise CgroupMemoryLimitError( + f"CI must set {COMPILE_MEMORY_LIMIT_ENV_VAR} so every timed compile runs in a " + "memory-limited transient systemd scope" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 801b9dc17d..c14d1cc038 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -34,6 +34,9 @@ import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class +from scripts.cold_compile_performance.main.compile_memory_scope_command import ( + compile_memory_scope_command, +) from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, @@ -726,6 +729,7 @@ def _run_timed_sqb( output_path: Path = project_dir.parent / f"{label}.out" stderr_path: Path = project_dir.parent / f"{label}.stderr" command: list[str] = [ + *compile_memory_scope_command(label=label), "/usr/bin/time", "--quiet", "--output", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 89601840a5..5558f90575 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -12,8 +12,8 @@ dense_test_path, write_dense_compile_project, ) -from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( - assert_required_cgroup_memory_limit, +from scripts.cold_compile_performance.main.require_compile_memory_scope import ( + require_compile_memory_scope, ) from sqlbuild._native import ( Rule, @@ -85,7 +85,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ test_case: DenseCompileGuardTestCase, tmp_path: Path, ) -> None: - assert_required_cgroup_memory_limit() + require_compile_memory_scope() project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) config: RulesConfig = load_rules_config(project_dir=project_dir) @@ -179,7 +179,7 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches test_case: DenseWarmEditCompileGuardTestCase, tmp_path: Path, ) -> None: - assert_required_cgroup_memory_limit() + require_compile_memory_scope() project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) edited_model_path: Path = dense_model_path( @@ -264,7 +264,7 @@ def test_given_dense_project_when_editing_one_file_then_delta_compiles_stay_chea test_case: DenseDeltaEditLoopGuardTestCase, tmp_path: Path, ) -> None: - assert_required_cgroup_memory_limit() + require_compile_memory_scope() project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) index: int = test_case.edited_model_index diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py index 57804a95e7..c8ed5bc49f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py @@ -10,8 +10,8 @@ import pytest -from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( - assert_required_cgroup_memory_limit, +from scripts.cold_compile_performance.main.require_compile_memory_scope import ( + require_compile_memory_scope, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( GeneratedBenchmarkCompileTestCase, @@ -44,7 +44,7 @@ def test_given_generated_duckshop_10k_when_compiling_cold_then_matches_checksum_and_is_clean( test_case: GeneratedBenchmarkCompileTestCase, tmp_path: Path ) -> None: - assert_required_cgroup_memory_limit() + require_compile_memory_scope() project_dir: Path = tmp_path / "duckshop_10k" generated: subprocess.CompletedProcess[str] = run_benchmark_generator(