ci: limit each timed dense compile to 4 GiB instead of the pytest tree (#1324)

The dense guard job ran make, uv and pytest inside one 4 GiB systemd unit,
so roughly 0.5 GiB of harness memory counted against a limit meant for the
compiler. Each timed sqb compile now runs in its own transient scope with
MemoryMax=4 GiB and no swap, and pytest runs outside it. Budgets are
unchanged. Under CI the guards refuse to run without the per-compile limit,
and a missing systemd-run or sudo fails naming the scope.

Author: kvlonge

.github/workflows/verify.yml

@@ -760,26 +760,14 @@ jobs:
         run: uv python install 3.12
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
-      - name: Run dense all-rules and delta edit-loop guards under 4 GiB
+      # Each timed compile runs in its own 4 GiB transient scope; pytest stays outside it.
+      - name: Run dense all-rules and delta edit-loop guards with each compile under 4 GiB
         env:
           BENCHMARK_MODELS: ${{ matrix.models }}
-          MEMORY_LIMIT_BYTES: "4294967296"
+          SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES: "4294967296"
         run: |
-          sudo systemd-run \
-            --unit="sqlbuild-dense-${BENCHMARK_MODELS}-${GITHUB_RUN_ID}" \
-            --uid="$(id -u)" \
-            --gid="$(id -g)" \
-            --same-dir \
-            --setenv="HOME=${HOME}" \
-            --setenv="PATH=${PATH}" \
-            --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \
-            --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \
-            --property="MemorySwapMax=0" \
-            --wait \
-            --collect \
-            --pipe \
-            make test-e2e-dense-compile-performance \
-              SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
+          make test-e2e-dense-compile-performance \
+            SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
 
       - name: Run diagnostic-heavy compile guards
         if: matrix.models == 1000

Makefile

@@ -334,7 +334,7 @@ test-e2e-dense-compile-performance:
 		-n auto --dist loadfile \
 		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \
 		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
-# DuckShop 10K runs after the dense guards, alone, so it never shares the memory limit with them.
+# DuckShop 10K runs after the dense guards, alone, so its compile never overlaps theirs.
 	if [ "$(SQLBUILD_BENCHMARK_MODELS)" = "10000" ]; then \
 		env PYTHONUNBUFFERED=1 uv run pytest benchmarks/duckshop-10k/tests -q --color=yes && \
 		env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \

fensu.toml

@@ -992,11 +992,21 @@ modules = ["scripts.cold_compile_performance.main.assert_required_cgroup_memory_
 symbols = ["assert_required_cgroup_memory_limit"]
 reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots."
 
+[[targets.python.dead_code.roots]]
+modules = ["scripts.cold_compile_performance.main.compile_memory_scope_command"]
+symbols = ["compile_memory_scope_command"]
+reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots."
+
 [[targets.python.dead_code.roots]]
 modules = ["scripts.cold_compile_performance.main.read_compile_measurement"]
 symbols = ["read_compile_measurement"]
 reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots."
 
+[[targets.python.dead_code.roots]]
+modules = ["scripts.cold_compile_performance.main.require_compile_memory_scope"]
+symbols = ["require_compile_memory_scope"]
+reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots."
+
 [[targets.python.dead_code.roots]]
 modules = ["scripts.cold_compile_performance.main.semantic_compile_fingerprint"]
 symbols = ["semantic_compile_fingerprint"]

scripts/cold_compile_performance/constants.py

@@ -1,6 +1,7 @@
 """Constants for fresh-process compile performance guards."""
 
 UNLIMITED_CGROUP_MEMORY_VALUE: str = "max"
+COMPILE_MEMORY_LIMIT_ENV_VAR: str = "SQLBUILD_COMPILE_MEMORY_LIMIT_BYTES"
 COMPILE_MEASUREMENT_FIELD_COUNT: int = 6
 
 DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99)

scripts/cold_compile_performance/main/compile_memory_scope_command.py

@@ -0,0 +1,42 @@
+"""Wrap one timed compile in a memory-limited transient systemd scope."""
+
+import os
+import shutil
+import uuid
+
+from scripts.cold_compile_performance.constants import COMPILE_MEMORY_LIMIT_ENV_VAR
+from scripts.cold_compile_performance.exceptions import CgroupMemoryLimitError
+
+
+def compile_memory_scope_command(*, label: str) -> tuple[str, ...]:
+    """Prefix for a scope with the configured per-compile limit; refuse to run without it."""
+
+    raw_limit: str | None = os.environ.get(COMPILE_MEMORY_LIMIT_ENV_VAR)
+    if raw_limit is None:
+        return ()
+    unit: str = f"sqlbuild-compile-{label}-{os.getpid()}-{uuid.uuid4().hex[:8]}"
+    systemd_run: str | None = shutil.which("systemd-run")
+    sudo: str | None = shutil.which("sudo")
+    caller_path_over_sudo_secure_path: str = f"PATH={os.environ.get('PATH', '')}"
+    if systemd_run is None or sudo is None:
+        raise CgroupMemoryLimitError(
+            f"{COMPILE_MEMORY_LIMIT_ENV_VAR}={raw_limit} requires systemd-run and sudo to start "
+            f"the transient scope {unit}.scope; refusing to compile without the memory limit"
+        )
+    return (
+        sudo,
+        "--non-interactive",
+        "--preserve-env",
+        systemd_run,
+        "--scope",
+        "--quiet",
+        "--collect",
+        f"--unit={unit}",
+        f"--uid={os.getuid()}",
+        f"--gid={os.getgid()}",
+        f"--property=MemoryMax={int(raw_limit)}",
+        "--property=MemorySwapMax=0",
+        "--",
+        "/usr/bin/env",
+        caller_path_over_sudo_secure_path,
+    )

scripts/cold_compile_performance/main/require_compile_memory_scope.py

@@ -0,0 +1,16 @@
+"""Require a per-compile memory limit when guards run in CI."""
+
+import os
+
+from scripts.cold_compile_performance.constants import COMPILE_MEMORY_LIMIT_ENV_VAR
+from scripts.cold_compile_performance.exceptions import CgroupMemoryLimitError
+
+
+def require_compile_memory_scope() -> None:
+    """Refuse to run in CI unless each timed compile gets its own memory-limited scope."""
+
+    if os.environ.get("CI") and COMPILE_MEMORY_LIMIT_ENV_VAR not in os.environ:
+        raise CgroupMemoryLimitError(
+            f"CI must set {COMPILE_MEMORY_LIMIT_ENV_VAR} so every timed compile runs in a "
+            "memory-limited transient systemd scope"
+        )

tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py

@@ -34,6 +34,9 @@
 import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings
 import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module
 import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class
+from scripts.cold_compile_performance.main.compile_memory_scope_command import (
+    compile_memory_scope_command,
+)
 from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement
 from scripts.cold_compile_performance.main.semantic_compile_fingerprint import (
     semantic_compile_fingerprint,
@@ -726,6 +729,7 @@ def _run_timed_sqb(
     output_path: Path = project_dir.parent / f"{label}.out"
     stderr_path: Path = project_dir.parent / f"{label}.stderr"
     command: list[str] = [
+        *compile_memory_scope_command(label=label),
         "/usr/bin/time",
         "--quiet",
         "--output",

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py

@@ -12,8 +12,8 @@
     dense_test_path,
     write_dense_compile_project,
 )
-from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import (
-    assert_required_cgroup_memory_limit,
+from scripts.cold_compile_performance.main.require_compile_memory_scope import (
+    require_compile_memory_scope,
 )
 from sqlbuild._native import (
     Rule,
@@ -85,7 +85,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
     test_case: DenseCompileGuardTestCase,
     tmp_path: Path,
 ) -> None:
-    assert_required_cgroup_memory_limit()
+    require_compile_memory_scope()
     project_dir: Path = tmp_path / "dense_orders"
     write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count)
     config: RulesConfig = load_rules_config(project_dir=project_dir)
@@ -179,7 +179,7 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches
     test_case: DenseWarmEditCompileGuardTestCase,
     tmp_path: Path,
 ) -> None:
-    assert_required_cgroup_memory_limit()
+    require_compile_memory_scope()
     project_dir: Path = tmp_path / "dense_orders"
     write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count)
     edited_model_path: Path = dense_model_path(
@@ -264,7 +264,7 @@ def test_given_dense_project_when_editing_one_file_then_delta_compiles_stay_chea
     test_case: DenseDeltaEditLoopGuardTestCase,
     tmp_path: Path,
 ) -> None:
-    assert_required_cgroup_memory_limit()
+    require_compile_memory_scope()
     project_dir: Path = tmp_path / "dense_orders"
     write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count)
     index: int = test_case.edited_model_index

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py

@@ -10,8 +10,8 @@
 
 import pytest
 
-from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import (
-    assert_required_cgroup_memory_limit,
+from scripts.cold_compile_performance.main.require_compile_memory_scope import (
+    require_compile_memory_scope,
 )
 from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import (
     GeneratedBenchmarkCompileTestCase,
@@ -44,7 +44,7 @@
 def test_given_generated_duckshop_10k_when_compiling_cold_then_matches_checksum_and_is_clean(
     test_case: GeneratedBenchmarkCompileTestCase, tmp_path: Path
 ) -> None:
-    assert_required_cgroup_memory_limit()
+    require_compile_memory_scope()
     project_dir: Path = tmp_path / "duckshop_10k"
 
     generated: subprocess.CompletedProcess[str] = run_benchmark_generator(