From 69f7a730af7c83e5520594afbab228c2545a64fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 22 Sep 2026 18:22:15 +0100 Subject: [PATCH] test: guard varied warm compilation and edits --- .github/workflows/verify.yml | 20 +++ Makefile | 8 +- docs/dense-compile-performance.md | 4 + docs/warm-compile-performance.md | 48 ++++++ .../_helpers/varied_benchmark.py | 117 +++++++++++++++ .../_helpers/varied_project.py | 138 +++++++++++++++++ .../_helpers/varied_statistics.py | 83 +++++++++++ scripts/cold_compile_performance/constants.py | 7 + .../cold_compile_performance/exceptions.py | 4 + scripts/cold_compile_performance/models.py | 15 ++ .../cli/commands/main/compile/_test_types.py | 19 +++ .../test_varied_compile_cache_performance.py | 139 ++++++++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 7 + .../main/test_varied_compile_fixture.py | 74 ++++++++++ 14 files changed, 682 insertions(+), 1 deletion(-) create mode 100644 docs/warm-compile-performance.md create mode 100644 scripts/cold_compile_performance/_helpers/varied_benchmark.py create mode 100644 scripts/cold_compile_performance/_helpers/varied_project.py create mode 100644 scripts/cold_compile_performance/_helpers/varied_statistics.py create mode 100644 scripts/cold_compile_performance/models.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_varied_compile_fixture.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 8827d87d94..d0af8bebd5 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -419,6 +419,26 @@ jobs: make test-e2e-cache-compile-performance \ SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" + - name: Run varied 1k warm-cache guard under 4 GiB + if: matrix.models == 3000 + env: + MEMORY_LIMIT_BYTES: "4294967296" + run: | + sudo systemd-run \ + --unit="sqlbuild-varied-cache-${GITHUB_RUN_ID}" \ + --uid="$(id -u)" \ + --gid="$(id -g)" \ + --same-dir \ + --setenv="HOME=${HOME}" \ + --setenv="PATH=${PATH}" \ + --setenv="SQLBUILD_REQUIRE_MEMORY_LIMIT_BYTES=${MEMORY_LIMIT_BYTES}" \ + --property="MemoryMax=${MEMORY_LIMIT_BYTES}" \ + --property="MemorySwapMax=0" \ + --wait \ + --collect \ + --pipe \ + make test-e2e-varied-cache-performance + rules_performance: name: Rules performance guards needs: [changes, lockfile] diff --git a/Makefile b/Makefile index 052054a51c..ef7d6133ad 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SHELL := /bin/bash test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \ test-e2e-duckdb-integrations test-e2e-performance \ test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ - test-e2e-dense-compile-performance + test-e2e-dense-compile-performance test-e2e-varied-cache-performance format: uv run ruff format . @@ -255,6 +255,12 @@ test-e2e-cache-compile-performance: -vv --log-cli-level=INFO --color=yes +test-e2e-varied-cache-performance: + env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ + tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py \ + -n auto --dist loadfile -m cache_compile_performance \ + -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes + test-virtual: @mkdir -p /tmp/opencode @log="/tmp/opencode/test-virtual-$$(date +%Y%m%d-%H%M%S).log"; \ diff --git a/docs/dense-compile-performance.md b/docs/dense-compile-performance.md index a8938ec2e4..deaf541b62 100644 --- a/docs/dense-compile-performance.md +++ b/docs/dense-compile-performance.md @@ -74,6 +74,10 @@ Custom rules continue to run in their isolated host. Disabled rules caches avoid unused fact fingerprints; enabled caches retain their normal identities and invalidation behavior. +The [varied warm-compile guard](warm-compile-performance.md) adds operator-shape +diversity and a shared dependency graph, measuring unchanged, leaf, upstream and +macro-edit compiles against uncached semantic/artifact oracles. + This fixture protects dense query analysis, declaration expansion, rules and artifact generation. It is not an exact reproduction of every application: dynamic plugins, incremental materialization mixes and deeply nested query diff --git a/docs/warm-compile-performance.md b/docs/warm-compile-performance.md new file mode 100644 index 0000000000..d349b27506 --- /dev/null +++ b/docs/warm-compile-performance.md @@ -0,0 +1,48 @@ +# Varied warm-compile guard + +`make test-e2e-varied-cache-performance` exercises a generated 1,000-model order +project in fresh CLI processes. The cache performance CI job runs this guard in a +4GiB cgroup with swap disabled, alongside the existing scaling benchmarks. + +The workload uses independently authored synthetic SQL. It retains the dense +fixture's sources, seeds, SQL functions, audits, scoped Python macros, SQL tests, +all built-in rules and one isolated custom project rule. It adds a shared dependency +graph and varies the actual expressions: CASE, aggregate windows, array indexing, +null handling, grouped queries, lateral joins and unions. + +## Structural protection + +The guard measures structure from compiled SQL and model dependencies. It requires: + +- At least 950 distinct operator-tree shapes. Identifier names and literal values + do not count toward this diversity measurement. +- A connected component containing at least 900 models. +- Dependency depth between 40 and 60 models, with 150–400 leaf models. + +These checks prevent a fixture change from silently replacing the workload with +mostly repeated queries or independent shallow chains. Structural diversity is +still a proxy for application workloads; private acceptance measurements remain +separate from this public synthetic fixture. + +## Cache and correctness protection + +The guard measures initial cache fill, an unchanged warm compile, a leaf SQL edit, +a shared upstream SQL edit and a Python macro edit. Each edit is followed by an +unchanged compile and an uncached oracle with both analysis and rule cache reuse +disabled. Project configuration is restored after each oracle. + +Every edited cached result must match its oracle's semantic JSON and every compiled +artifact. Unchanged runs must reuse analysis for every model. Edit runs must record +both reuse and invalidation, rather than treating a high cache-hit count alone as +evidence of correctness. A smaller integration fixture executes its generated SQL +tests in DuckDB with the standard rules enabled. + +Warm and edit budgets apply to complete fresh-process CLI wall time, including +discovery, macro expansion, lint, rules, artifacts and output. SQL-analysis timing +alone is insufficient: these other phases can dominate after analysis is cached. + +The initial ceilings are 24s for cache fill, 10s for unchanged warm compiles and +13s for edits, with a 2GiB process-RSS ceiling and the separate 4GiB combined CI +limit. These are regression ceilings with hosted-execution headroom, not promised +application timings. Golden fingerprints also protect the baseline and edited +outputs across compiler releases. diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py new file mode 100644 index 0000000000..a76454d163 --- /dev/null +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -0,0 +1,117 @@ +"""Measure diverse cached compiler workloads against uncached semantic oracles.""" + +from itertools import filterfalse +from operator import attrgetter +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import dense_model_name +from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project +from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError +from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules +from sqlbuild.rule_engine.main.load_config import load_rules_config +from sqlbuild.rule_engine.models import Rule, RulesConfig +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + FreshProcessCompileBenchmarkResult, + _run_fresh_process_compile_benchmark, +) + + +def run_varied_cache_benchmark( + *, project_dir: Path, test_case: VariedCompileCacheTestCase +) -> dict[str, FreshProcessCompileBenchmarkResult]: + write_varied_compile_project(project_dir=project_dir, model_count=test_case.model_count) + _assert_complete_rules(project_dir) + measurements: dict[str, FreshProcessCompileBenchmarkResult] = {} + measurements["cold"] = _measure( + project_dir=project_dir, label="cold", budget=test_case.expected_cold_max_seconds + ) + measurements["warm"] = _measure( + project_dir=project_dir, label="warm", budget=test_case.expected_warm_max_seconds + ) + leaf: Path = next( + (project_dir / "models").rglob(f"{dense_model_name(test_case.model_count - 1)}.sql") + ) + _edit_model(leaf) + measurements.update(_measure_edit(project_dir=project_dir, label="leaf", test_case=test_case)) + shared: Path = next((project_dir / "models").rglob(f"{dense_model_name(0)}.sql")) + _edit_model(shared) + measurements.update(_measure_edit(project_dir=project_dir, label="shared", test_case=test_case)) + macro: Path = next((project_dir / "models").rglob("offset00000.py")) + contents: str = macro.read_text(encoding="utf-8") + if contents.count('return f"({expression} + 1)"') != 1: + raise CompileBenchmarkFixtureError("Expected one editable macro expression") + macro.write_text( + contents.replace('return f"({expression} + 1)"', 'return f"({expression} + 2)"'), + encoding="utf-8", + ) + measurements.update(_measure_edit(project_dir=project_dir, label="macro", test_case=test_case)) + return measurements + + +def _assert_complete_rules(project_dir: Path) -> None: + config: RulesConfig = load_rules_config(project_dir=project_dir) + catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) + selected: tuple[Rule, ...] = select_rules( + catalogue=catalogue, config=config, project_dir=project_dir + ) + builtin_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), catalogue)} + selected_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), selected)} + if ( + selected_codes != builtin_codes + or sum(rule.custom for rule in selected) != 1 + or config.ignore + or config.rule_exceptions + or config.rule_ignores + or config.thresholds + ): + raise CompileBenchmarkFixtureError( + "Expected all built-in rules and one unrelaxed custom rule" + ) + + +def _edit_model(path: Path) -> None: + contents: str = path.read_text(encoding="utf-8") + if contents.count("CAST(id AS INTEGER)") != 1: + raise CompileBenchmarkFixtureError("Expected one editable model projection") + path.write_text( + contents.replace("CAST(id AS INTEGER)", "CAST(COALESCE(id, id) AS INTEGER)"), + encoding="utf-8", + ) + + +def _measure_edit( + *, + project_dir: Path, + label: str, + test_case: VariedCompileCacheTestCase, +) -> dict[str, FreshProcessCompileBenchmarkResult]: + measurements: dict[str, FreshProcessCompileBenchmarkResult] = {} + measurements[label] = _measure( + project_dir=project_dir, label=label, budget=test_case.expected_edit_max_seconds + ) + measurements[f"after_{label}"] = _measure( + project_dir=project_dir, label=f"after_{label}", budget=test_case.expected_warm_max_seconds + ) + config: Path = project_dir / "sqlbuild_project.toml" + original: bytes = config.read_bytes() + try: + config.write_bytes(original + b"\n[rules.cache]\nenabled = false\n") + measurements[f"oracle_{label}"] = _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label=f"oracle-{label}", + expected_max_wall_seconds=test_case.expected_cold_max_seconds, + compile_args=("--no-cache",), + ) + finally: + config.write_bytes(original) + return measurements + + +def _measure(*, project_dir: Path, label: str, budget: float) -> FreshProcessCompileBenchmarkResult: + return _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label=f"varied-{label}", + expected_max_wall_seconds=budget, + compile_args=(), + ) diff --git a/scripts/cold_compile_performance/_helpers/varied_project.py b/scripts/cold_compile_performance/_helpers/varied_project.py new file mode 100644 index 0000000000..7ef058e0e3 --- /dev/null +++ b/scripts/cold_compile_performance/_helpers/varied_project.py @@ -0,0 +1,138 @@ +"""Generate diverse order-query shapes over a shared, bounded dependency graph.""" + +import re +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import ( + dense_model_name, + write_dense_compile_project, +) +from scripts.cold_compile_performance.constants import ( + VARIED_ARRAY_VARIANT, + VARIED_CASE_SHARE, + VARIED_GROUP_SHARE, + VARIED_WINDOW_SHARE, + VARIED_WINDOW_VARIANTS, +) + +_QUANTITY_EXPRESSION: str = "COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)" +_QUANTITY_VARIANTS: tuple[str, ...] = ( + _QUANTITY_EXPRESSION, + "CASE WHEN b.id IS NOT NULL THEN b.amount + b.id END", + "ABS(b.amount) + ABS(b.id)", + "SUM(b.amount) OVER (PARTITION BY b.id) + b.id", + "MAX(b.amount) OVER (PARTITION BY b.id) + b.id", + "COALESCE(NULLIF(b.amount, 0), b.id) + b.id", + "[b.amount, b.amount][1] + b.id", + "GREATEST(b.amount, b.id) + LEAST(b.amount, b.id)", +) +_IMPORT: re.Pattern[str] = re.compile(r'__(?:ref|source)\("([^"\n]+)"\)') + + +def write_varied_compile_project(*, project_dir: Path, model_count: int) -> None: + """Write independent neutral SQL with diverse expressions and shared ancestors.""" + + write_dense_compile_project(project_dir=project_dir, model_count=model_count) + for index, model in enumerate(sorted((project_dir / "models").rglob("*.sql"))): + contents: str = model.read_text(encoding="utf-8") + replacements: dict[str, str] = _dependency_replacements( + index=index, contents=contents, model_count=model_count + ) + for previous, current in replacements.items(): + contents = contents.replace(previous, current) + contents = _vary_expressions(index=index, contents=contents) + contents = _vary_query_structure(index=index, contents=contents) + model.write_text(contents, encoding="utf-8") + test_name: str = ( + f"test_{dense_model_name(index).replace('__', '_')}__preserves_order_id.sql" + ) + test: Path = next((project_dir / "tests").rglob(test_name)) + test_sql: str = test.read_text(encoding="utf-8") + for previous, current in replacements.items(): + test_sql = test_sql.replace(_mock_name(previous), _mock_name(current)) + test.write_text(test_sql, encoding="utf-8") + + +def _vary_expressions(*, index: int, contents: str) -> str: + fragments: list[str] = contents.split(_QUANTITY_EXPRESSION) + output: list[str] = [fragments[0]] + for position, fragment in enumerate(fragments[1:]): + selector: int = ((index + 1) * 2654435761) ^ ((position + 1) * 2246822519) + variant: int = (selector >> ((position + 1) % 19)) % len(_QUANTITY_VARIANTS) + if variant == VARIED_ARRAY_VARIANT and index % 5 != 0: + variant = 0 + if variant in VARIED_WINDOW_VARIANTS and index % 20 >= VARIED_WINDOW_SHARE: + variant = 2 + if variant == 1 and index % 10 >= VARIED_CASE_SHARE: + variant = 5 + output.extend((_QUANTITY_VARIANTS[variant], fragment)) + return "".join(output) + + +def _vary_query_structure(*, index: int, contents: str) -> str: + if index % 4 != 1: + contents = re.sub( + r"combined AS \(\n(SELECT [^\n]+ FROM projected)\n.*?\n\)", + r"combined AS (\n\1\n)", + contents, + flags=re.DOTALL, + ) + prefix, marker, tail = contents.partition("projected AS (") + parts: tuple[str, str, str] = tail.partition("),\ncombined AS (") + projected: str = parts[0] + separator: str = parts[1] + rest: str = parts[2] + if index % 7 == 0: + projected = projected.replace("b.amount", "quantities.quantity", 1) + projected += ( + "\nINNER JOIN LATERAL (SELECT b.amount AS quantity) AS quantities" + " ON quantities.quantity = b.amount" + ) + if index % 10 < VARIED_GROUP_SHARE: + keys: list[str] = ["b.id", "b.amount"] + if index % 7 == 0: + keys.append("quantities.quantity") + keys.extend(re.findall(r"COALESCE\((l\d+\.amount), 0\)", projected)) + projected += "\nGROUP BY " + ", ".join(dict.fromkeys(keys)) + return prefix + marker + projected + separator + rest + + +def _dependency_replacements(*, index: int, contents: str, model_count: int) -> dict[str, str]: + cohort_start: int = index // 1000 * 1000 + local_index: int = index - cohort_start + cohort_size: int = min(1000, model_count - cohort_start) + roots: int = max(1, cohort_size * 14 // 100) + width: int = max(1, cohort_size * 17 // 1000) + source_count: int = max(1, cohort_size * 24 // 100) + imports: list[str] = list(dict.fromkeys(match.group(0) for match in _IMPORT.finditer(contents))) + replacements: dict[str, str] = {} + used: set[int] = set() + for position, imported in enumerate(imports): + if local_index < roots: + replacements[imported] = ( + f'__source("orders_{cohort_start * 24 // 100 + local_index % source_count:05d}")' + if position == 0 + else imported + ) + elif position == 0: + level: int = (local_index - roots) // width + start: int = roots + (level - 1) * width if level else 0 + count: int = width if level else roots + slot: int = local_index - local_index % 3 if local_index % 3 == 1 else local_index + parent: int = cohort_start + start + (slot * 7 + 3) % count + used.add(parent) + replacements[imported] = f'__ref("{dense_model_name(parent)}")' + elif position % 2 == 0: + parent = cohort_start + (local_index * 31 + position * 13) % roots + if parent not in used: + used.add(parent) + replacements[imported] = f'__ref("{dense_model_name(parent)}")' + return replacements + + +def _mock_name(reference: str) -> str: + return ( + reference.replace('__ref("', "__ref__") + .replace('__source("', "__source__") + .removesuffix('")') + ) diff --git a/scripts/cold_compile_performance/_helpers/varied_statistics.py b/scripts/cold_compile_performance/_helpers/varied_statistics.py new file mode 100644 index 0000000000..2068451e3d --- /dev/null +++ b/scripts/cold_compile_performance/_helpers/varied_statistics.py @@ -0,0 +1,83 @@ +"""Measure workload structure independently of identifiers and SQL literal values.""" + +import re +from typing import Any, cast + +from scripts.cold_compile_performance.constants import ( + VARIED_ARRAY_NODE_KEYS, + VARIED_WINDOW_NODE_FRAGMENT, +) +from scripts.cold_compile_performance.models import VariedProjectStatistics +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + +_UDF_REFERENCE: re.Pattern[str] = re.compile(r'__udf\("([a-zA-Z0-9_]+)"\)') + + +def varied_project_statistics(payload: dict[str, object]) -> VariedProjectStatistics: + resources: dict[str, object] = cast(dict[str, object], payload["resources"]) + models: list[dict[str, Any]] = cast(list[dict[str, Any]], resources["models"]) + names: set[str] = {model["name"] for model in models} + dependencies: dict[str, set[str]] = {} + for model in models: + dependencies[model["name"]] = { + reference["name"] + for reference in model["depends_on"] + if reference["resource_type"] == CompiledResourceType.MODEL + } + shapes: set[tuple[str, ...]] = set() + array_models: int = 0 + window_models: int = 0 + for model in models: + query_sql: str = _UDF_REFERENCE.sub(r"\1", model["query_sql"]) + expression: Any = import_polyglot_sql().parse_one(query_sql, dialect="snowflake") + shape: tuple[str, ...] = tuple(str(node.key) for node in expression.walk()) + shapes.add(shape) + array_models += int(any(key in VARIED_ARRAY_NODE_KEYS for key in shape)) + window_models += int(any(VARIED_WINDOW_NODE_FRAGMENT in key for key in shape)) + depths: dict[str, int] = {} + for name in names: + _depth(name=name, dependencies=dependencies, depths=depths) + consumed: set[str] = set().union(*dependencies.values()) + return VariedProjectStatistics( + model_count=len(models), + distinct_operator_shapes=len(shapes), + model_edges=sum(len(parents) for parents in dependencies.values()), + largest_component=_largest_component(dependencies), + maximum_depth=max(depths.values(), default=0), + leaf_models=len(names - consumed), + array_models=array_models, + window_models=window_models, + ) + + +def _depth(*, name: str, dependencies: dict[str, set[str]], depths: dict[str, int]) -> int: + if name not in depths: + depths[name] = 1 + max( + ( + _depth(name=parent, dependencies=dependencies, depths=depths) + for parent in dependencies[name] + ), + default=0, + ) + return depths[name] + + +def _largest_component(dependencies: dict[str, set[str]]) -> int: + neighbors: dict[str, set[str]] = {name: set(parents) for name, parents in dependencies.items()} + for name, parents in dependencies.items(): + for parent in parents: + neighbors[parent].add(name) + unseen: set[str] = set(neighbors) + largest: int = 0 + while unseen: + pending: list[str] = [unseen.pop()] + size: int = 0 + while pending: + current: str = pending.pop() + size += 1 + discovered: set[str] = neighbors[current] & unseen + unseen.difference_update(discovered) + pending.extend(discovered) + largest = max(largest, size) + return largest diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index a8599c5db8..2e7ef3ed24 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -8,6 +8,13 @@ DENSE_UNION_MEDIUM_INDEX: int = 7 DENSE_MACRO_SHARE: int = 3 DENSE_MACRO_QUANTITY_COLUMNS: int = 2 +VARIED_ARRAY_VARIANT: int = 6 +VARIED_WINDOW_VARIANTS: frozenset[int] = frozenset({3, 4}) +VARIED_WINDOW_SHARE: int = 7 +VARIED_CASE_SHARE: int = 3 +VARIED_GROUP_SHARE: int = 3 +VARIED_ARRAY_NODE_KEYS: frozenset[str] = frozenset({"array_func", "subscript"}) +VARIED_WINDOW_NODE_FRAGMENT: str = "window" DENSE_AMOUNT_AUDIT_SHARE: int = 7 DENSE_DEEP_CTE_PERCENTILE: int = 97 DENSE_MEDIUM_CTE_PERCENTILE: int = 80 diff --git a/scripts/cold_compile_performance/exceptions.py b/scripts/cold_compile_performance/exceptions.py index d2359833bf..99cae61bbe 100644 --- a/scripts/cold_compile_performance/exceptions.py +++ b/scripts/cold_compile_performance/exceptions.py @@ -3,3 +3,7 @@ class CgroupMemoryLimitError(RuntimeError): """Raised when a required compile memory limit is not active.""" + + +class CompileBenchmarkFixtureError(RuntimeError): + """Raised when a generated workload no longer satisfies its benchmark contract.""" diff --git a/scripts/cold_compile_performance/models.py b/scripts/cold_compile_performance/models.py new file mode 100644 index 0000000000..9118eaf908 --- /dev/null +++ b/scripts/cold_compile_performance/models.py @@ -0,0 +1,15 @@ +"""Structural measurements for generated compiler performance workloads.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class VariedProjectStatistics: + model_count: int + distinct_operator_shapes: int + model_edges: int + largest_component: int + maximum_depth: int + leaf_models: int + array_models: int + window_models: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 62c8f3a354..055e8d7753 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -5,6 +5,25 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class VariedCompileCacheTestCase: + description: str + model_count: int + expected_cold_max_seconds: float + expected_warm_max_seconds: float + expected_edit_max_seconds: float + expected_max_rss_bytes: int + expected_min_operator_shapes: int + expected_min_component: int + expected_min_depth: int + expected_max_depth: int + expected_min_leaves: int + expected_max_leaves: int + expected_min_array_models: int + expected_min_window_models: int + expected_fingerprints: tuple[str, str, str, str] + + @dataclass(frozen=True) class CompilePerformanceGuardTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py new file mode 100644 index 0000000000..275fc1b114 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -0,0 +1,139 @@ +"""Fresh-process warm and edit budgets for structurally diverse all-rules SQL.""" + +import logging +from pathlib import Path +from typing import cast + +import pytest + +from scripts.cold_compile_performance._helpers.varied_benchmark import ( + run_varied_cache_benchmark, +) +from scripts.cold_compile_performance._helpers.varied_statistics import varied_project_statistics +from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( + assert_required_cgroup_memory_limit, +) +from scripts.cold_compile_performance.models import VariedProjectStatistics +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + FreshProcessCompileBenchmarkResult, + fresh_process_compile_cache_metrics, +) + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +@pytest.mark.performance +@pytest.mark.cache_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + VariedCompileCacheTestCase( + "models_1000_varied_shared_graph", + 1000, + 24.0, + 10.0, + 13.0, + 2 * 1024**3, + 950, + 900, + 40, + 60, + 150, + 400, + 150, + 250, + ( + "1c107624785e798ad43bc271ce90f60ed62ed703fe01a240687dec54b8080e15", + "525245fb8374d566c8ff7a111fb0635abdc29f6ae25faba2a0f8b93beeeef8cc", + "2e27ca67a51cbb0672794518ab611ab21a1730b294d7a0252e72d6ca6322fe5b", + "be3f058013b62f2b0d54d6a1d52f7232d12c5c36641d801003ad8fda0b46e11f", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_varied_shared_graph_when_compiling_warm_and_edits_then_budgets_and_oracles_hold( + test_case: VariedCompileCacheTestCase, + tmp_path: Path, +) -> None: + assert_required_cgroup_memory_limit() + measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_varied_cache_benchmark( + project_dir=tmp_path / "varied_orders", + test_case=test_case, + ) + statistics: VariedProjectStatistics = varied_project_statistics(measurements["cold"].payload) + _LOGGER.info("varied project structure: %s", statistics) + assert statistics.model_count == test_case.model_count + assert statistics.distinct_operator_shapes >= test_case.expected_min_operator_shapes + assert statistics.largest_component >= test_case.expected_min_component + assert test_case.expected_min_depth <= statistics.maximum_depth <= test_case.expected_max_depth + assert test_case.expected_min_leaves <= statistics.leaf_models <= test_case.expected_max_leaves + assert statistics.array_models >= test_case.expected_min_array_models + assert statistics.window_models >= test_case.expected_min_window_models + for label, result in measurements.items(): + _LOGGER.info( + "varied cache path=%s wall=%.3fs peak_rss_bytes=%d fingerprint=%s metrics=%s", + label, + result.elapsed_seconds, + result.peak_rss_bytes, + result.semantic_fingerprint, + fresh_process_compile_cache_metrics(result), + ) + assert result.payload["diagnostics"] == [] + assert result.payload["has_errors"] is False + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes + assert measurements["cold"].semantic_fingerprint == measurements["warm"].semantic_fingerprint + assert measurements["cold"].elapsed_seconds < test_case.expected_cold_max_seconds + assert fresh_process_compile_cache_metrics(measurements["cold"]) == ( + 0, + 0, + test_case.model_count, + 0, + ) + cold_timings: dict[str, int] = cast( + dict[str, int], measurements["cold"].payload["compile_timings"] + ) + assert cold_timings["rule_cache_hits"] == 0 + assert ( + tuple( + measurements[label].semantic_fingerprint + for label in ("cold", "leaf", "shared", "macro") + ) + == test_case.expected_fingerprints + ) + for label in ("warm", "after_leaf", "after_shared", "after_macro"): + assert measurements[label].elapsed_seconds < test_case.expected_warm_max_seconds + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics( + measurements[label] + ) + assert batch_hits + entry_hits == test_case.model_count + assert misses == bypasses == 0 + timings: dict[str, int] = cast( + dict[str, int], measurements[label].payload["compile_timings"] + ) + assert timings["rule_cache_misses"] == 0 + assert timings["rule_cache_hits"] > 0 + for label in ("leaf", "shared", "macro"): + result: FreshProcessCompileBenchmarkResult = measurements[label] + assert result.elapsed_seconds < test_case.expected_edit_max_seconds + assert result.semantic_fingerprint == measurements[f"after_{label}"].semantic_fingerprint + assert result.semantic_fingerprint == measurements[f"oracle_{label}"].semantic_fingerprint + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(result) + assert 0 < misses < test_case.model_count + assert batch_hits + entry_hits + misses == test_case.model_count + assert bypasses == 0 + assert fresh_process_compile_cache_metrics(measurements[f"oracle_{label}"]) == ( + 0, + 0, + 0, + test_case.model_count, + ) + oracle_timings: dict[str, int] = cast( + dict[str, int], measurements[f"oracle_{label}"].payload["compile_timings"] + ) + assert oracle_timings["rule_cache_hits"] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 4ff7fd72b7..73dec1c303 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -7,6 +7,13 @@ from sqlbuild.compiler.planner.models import CursorOverrides +@dataclass(frozen=True) +class VariedCompileFixtureTestCase: + description: str + model_count: int + expected_exit_code: int + + @dataclass(frozen=True) class PreparedArtifactsCompileTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_varied_compile_fixture.py b/tests/integration/src/sqlbuild/cli/commands/main/test_varied_compile_fixture.py new file mode 100644 index 0000000000..46b46ddaef --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_varied_compile_fixture.py @@ -0,0 +1,74 @@ +"""Execute diverse generated queries and verify unchanged-cache equivalence.""" + +import json +from pathlib import Path + +import pytest + +from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + VariedCompileFixtureTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + (VariedCompileFixtureTestCase("varied windows arrays and shared dependencies", 64, 0),), + ids=lambda case: case.description, +) +def test_given_varied_fixture_when_compiling_warm_and_executing_tests_then_results_match( + test_case: VariedCompileFixtureTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project: Path = tmp_path / "orders" + write_varied_compile_project(project_dir=project, model_count=test_case.model_count) + config: Path = project / "sqlbuild_project.toml" + config.write_text( + config.read_text(encoding="utf-8") + .replace('adapter = "snowflake"', 'adapter = "duckdb"') + .replace('database = "warehouse"\n', "") + .replace('schema = "analytics"', 'schema = "main"') + + f'\n[connection]\ndatabase = "{project / "orders.duckdb"}"\n', + encoding="utf-8", + ) + arguments: list[str] = ["--project-dir", str(project), "compile", "--json"] + cold_exit: int = main(arguments) + cold_output: str = capsys.readouterr().out + assert cold_exit == test_case.expected_exit_code, cold_output + cold: dict = json.loads(cold_output) + compiled: Path = project / "target" / "compiled" + artifacts: dict[Path, bytes] = {path: path.read_bytes() for path in compiled.rglob("*.sql")} + warm_exit: int = main(arguments) + warm_output: str = capsys.readouterr().out + assert warm_exit == test_case.expected_exit_code, warm_output + warm: dict = json.loads(warm_output) + assert cold["resources"] == warm["resources"] + assert cold["diagnostics"] == warm["diagnostics"] == [] + assert ( + warm["compile_timings"]["analysis_batch_cache_hits"] + + warm["compile_timings"]["analysis_entry_cache_hits"] + ) == test_case.model_count + assert warm["compile_timings"]["analysis_cache_misses"] == 0 + assert {path: path.read_bytes() for path in compiled.rglob("*.sql")} == artifacts + built: int = main( + [ + "--project-dir", + str(project), + "build", + "--select", + "fn_00000", + "--no-tests", + "--no-audits", + ] + ) + captured: str = "".join(capsys.readouterr()) + assert built == test_case.expected_exit_code, captured + tested: int = main(["--project-dir", str(project), "test"]) + captured = "".join(capsys.readouterr()) + assert tested == test_case.expected_exit_code, captured + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"]))