From f344a3362be812a37931588cc249061a096a554e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 19 Sep 2026 13:49:19 +0100 Subject: [PATCH] perf: reduce semantic compile analysis cost --- Cargo.lock | 4 +- Cargo.toml | 2 +- .../compile/_helpers/analysis/cte_facts.py | 18 +- .../main/compile/COMPILE_BENCHMARKS.md | 55 ++++ .../main/compile/DAGSTER_SHAPED_BENCHMARK.md | 44 --- .../cli/commands/main/compile/_test_types.py | 20 ++ .../cli/commands/main/compile/helpers.py | 307 +++++++++++++++++- .../main/compile/test_compile_performance.py | 85 ++++- 8 files changed, 472 insertions(+), 63 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/DAGSTER_SHAPED_BENCHMARK.md diff --git a/Cargo.lock b/Cargo.lock index 42d1531ad4..6df50aacb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,7 +280,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "polyglot-sql" version = "0.9.3" -source = "git+https://github.com/chio-labs/polyglot?rev=1307cb6a2ead0bd9bdfe1fb82887f37e42c291bb#1307cb6a2ead0bd9bdfe1fb82887f37e42c291bb" +source = "git+https://github.com/chio-labs/polyglot?rev=afd2c73a2fe40f59e275aa918264c41ae83cbe94#afd2c73a2fe40f59e275aa918264c41ae83cbe94" dependencies = [ "polyglot-sql-ast-derive", "serde", @@ -293,7 +293,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.9.3" -source = "git+https://github.com/chio-labs/polyglot?rev=1307cb6a2ead0bd9bdfe1fb82887f37e42c291bb#1307cb6a2ead0bd9bdfe1fb82887f37e42c291bb" +source = "git+https://github.com/chio-labs/polyglot?rev=afd2c73a2fe40f59e275aa918264c41ae83cbe94#afd2c73a2fe40f59e275aa918264c41ae83cbe94" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c4ad0fac0a..79d15ee652 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ fensu-policy = "=0.13.0" globset = "0.4" proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "1307cb6a2ead0bd9bdfe1fb82887f37e42c291bb", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot", rev = "afd2c73a2fe40f59e275aa918264c41ae83cbe94", default-features = false, features = [ "ast-tools", "builder", "dialect-bigquery", diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cte_facts.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cte_facts.py index 76726a3d3a..827ec87d9d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cte_facts.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/cte_facts.py @@ -456,11 +456,7 @@ def _polyglot_non_null_filter_context( select: Any, column_nullability_by_table: dict[str, dict[str, InferredNullability]], ) -> NonNullFilterContext | None: - select_args: object = getattr(select, "args", None) - if not isinstance(select_args, dict): - return None - select_args_dict: dict[str, object] = cast(dict[str, object], select_args) - where_payload: object = select_args_dict.get(_POLYGLOT_PAYLOAD_WHERE_CLAUSE) + where_payload: object = select.arg(_POLYGLOT_PAYLOAD_WHERE_CLAUSE) if not isinstance(where_payload, dict): return None predicate: object = cast(dict[str, object], where_payload).get(_POLYGLOT_PAYLOAD_THIS) @@ -616,7 +612,7 @@ def _polyglot_star_output_types( def _polyglot_top_level_ctes(root: Any) -> tuple[tuple[str, Any, bool], ...]: - with_payload: object = getattr(root, "args", {}).get("with") + with_payload: object = root.arg("with") raw_ctes: object = ( cast(dict[str, object], with_payload).get("ctes") if isinstance(with_payload, dict) @@ -678,10 +674,10 @@ def _polyglot_set_operation_output_types( inference_profile: ExpressionInferenceProfile, expression_type_resolver: _ExpressionTypeResolver, ) -> dict[str, str]: - if bool(getattr(operation, "args", {}).get("by_name")): + if bool(operation.arg("by_name")): branch_types: list[dict[str, str]] = [] for side in ("left", "right"): - branch: Any = getattr(operation, "args", {}).get(side) + branch: Any = operation.arg(side) if branch is None: return {} branch_types.append( @@ -707,7 +703,7 @@ def _polyglot_set_operation_output_types( } branch_slots: list[tuple[tuple[str, str | None], ...]] = [] for side in ("left", "right"): - branch: Any = getattr(operation, "args", {}).get(side) + branch: Any = operation.arg(side) if branch is None: return {} inferred: tuple[tuple[str, str | None], ...] = _polyglot_select_output_type_slots( @@ -751,7 +747,7 @@ def _polyglot_select_output_type_slots( if str(getattr(select, "kind", "")) in _POLYGLOT_SET_OPERATION_KINDS: branch_slots: list[tuple[tuple[str, str | None], ...]] = [] for side in ("left", "right"): - branch: Any = getattr(select, "args", {}).get(side) + branch: Any = select.arg(side) if branch is None: return () slots: tuple[tuple[str, str | None], ...] = _polyglot_select_output_type_slots( @@ -764,7 +760,7 @@ def _polyglot_select_output_type_slots( return () branch_slots.append(slots) left_slots, right_slots = branch_slots - if bool(getattr(select, "args", {}).get("by_name")): + if bool(select.arg("by_name")): right_types: dict[str, str | None] = dict(right_slots) return tuple( ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md new file mode 100644 index 0000000000..2d271c0008 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -0,0 +1,55 @@ +# Compile performance benchmarks + +SQLBuild maintains complementary generated projects because model count alone does not predict +compile cost. Every fixture uses neutral orders-style names and synthetic SQL. + +## Layered scale guard + +The layered guard combines 976 models, 232 sources, 46 seeds, 23 SQL functions, 700 attached +audits, 130 native tests, two hooks, and 54 execution layers. Its model files follow a bounded SQL +size distribution up to roughly 520 KB. The graph contains a 54-layer spine and independent +eight-model chains. + +This guard measures cold, unchanged, leaf-model edit, central-model edit, test edit, macro edit, +and project-config edit paths. It protects mixed-resource scaling and cache invalidation while the +separate 3,000/10,000-model and test-heavy guards retain their narrower contracts. + +Budgets: + +- cold compile: under 7 seconds; +- unchanged warm compile: under 3 seconds; +- one-model, one-test, and one-macro edits: under 4 seconds; +- project-config edit: under 8 seconds. + +## Semantic density guard + +The semantic guard exercises costs that SQL-size padding cannot represent: + +| Characteristic | Generated guard | +| --- | ---: | +| Models | 976 | +| Sources | 232 | +| Seeds | 46 | +| SQL functions | 23 | +| Attached audits | 1,645 | +| Native test cases | 958 | +| Declared model columns | about 33,000 | +| Model SQL | about 5.8 MB | +| Compiled test SQL | about 13 MB | +| Hooks | 2 | +| Execution layers | 54 | + +Declared column widths range from three to 538 columns. The SQL uses real expression trees rather +than comment-only padding: casts, conditional expressions, CTEs, derived tables, macros, functions, +seed joins, assertions, and a 267-branch set operation feeding a 270-column aggregate. The large set +operation specifically protects schema validation and CTE type inference from nested-tree +materialization regressions. + +The guard records phase timings for cold and unchanged warm compiles. It enforces a 60-second cold +budget and a 12-second warm budget. Resource counts, declared-column count, model SQL bytes, and +compiled-test bytes are asserted so simplifying the fixture cannot make the performance check pass. + +Both guards initialize the process-local SQL runtime with an unrelated 32-model project before +measurement. “Cold” means the measured project has no compiler cache or target artifacts; it does +not include one-time lazy initialization noise from the Python test worker. Phase timings are +attribution spans rather than additive buckets. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/DAGSTER_SHAPED_BENCHMARK.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/DAGSTER_SHAPED_BENCHMARK.md deleted file mode 100644 index 8a3a3c3958..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/DAGSTER_SHAPED_BENCHMARK.md +++ /dev/null @@ -1,44 +0,0 @@ -# Dagster-shaped compile benchmark - -This generated project protects SQLBuild against regressions that only appear when model, -declaration, native-test, and artifact-writing workloads occur together. It contains no Dagster -SQL, names, or fixture data. - -The profile was sampled from the September 2026 Dagster SQLBuild project. Counts and model SQL -sizes are intentionally close, while generated names and logic are generic: - -| Characteristic | Dagster profile | Generated guard | -| --- | ---: | ---: | -| Models | 976 | 976 | -| Sources | 232 | 232 | -| Seeds | 46 | 46 | -| SQL functions | 23 | 23 | -| Attached audits | 731 | 700 | -| Native test cases | 130 | 130 | -| Hooks | 2 | 2 | -| Execution layers | 54 | 54 | -| Model SQL bytes | 6.05 MB | about 6.4 MB | -| Model SQL p50 / p75 | 1.9 / 4.5 KB | 1.9 / 4.5 KB | -| Model SQL p90 / p95 | 10.9 / 15.2 KB | 11.0 / 15.2 KB | -| Model SQL p99 / max | 48.2 / 519 KB | 48.0 / 520 KB | - -The graph combines one 54-layer spine with bounded eight-model test chains. SQL shapes include -roughly 49 top-level CTE models, 49 derived-table models, and direct selects for the remainder. -It also includes path defaults, a reusable schema, an enforced contract, attached audits, two SQL -hooks, SQL functions, seed and source references, macros (including a composed macro), assertions, -26 repeated test targets, and fixtures ranging from 40 to 200 rows. Generated comments reproduce -the source-size distribution without manufacturing thousands of expensive CASE branches that do -not represent the real project's SQL analysis cost. - -The guard measures cold, unchanged, leaf-model edit, central-model edit, test edit, macro edit, and -project-config edit paths. JSON output reports discovery, attachment, model analysis, test input -compilation, test planning, comparison rendering, cache publication, physical writes, and stale -artifact traversal so a failure identifies the regressed phase rather than only the total runtime. -The detailed timings are attribution spans, not additive buckets: attachment includes test-input -compilation and its CTE-cache publication, model analysis includes its cache publication, and the -legacy write span contains test planning, comparison rendering, cache publication, physical writes, -and stale traversal. - -The process-local SQL runtime is initialized with an unrelated 32-model project before measuring. -“Cold” therefore means no compiler cache or target artifacts exist for the representative project; -it does not include one-time lazy initialization noise from the Python test worker. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8aa27f8058..b52d8e3ebc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -70,6 +70,26 @@ class LayeredProductionCompilePerformanceGuardTestCase: expected_config_edit_max_seconds: float +@dataclass(frozen=True) +class SemanticCompilePerformanceGuardTestCase: + description: str + model_count: int + source_count: int + seed_count: int + function_count: int + macro_count: int + test_count: int + audit_count: int + expected_min_declared_columns: int + expected_max_declared_columns: int + expected_min_model_sql_bytes: int + expected_max_model_sql_bytes: int + expected_min_compiled_test_bytes: int + expected_max_compiled_test_bytes: int + expected_cold_max_seconds: float + expected_warm_max_seconds: float + + @dataclass(frozen=True) class NamespaceCompileTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 5ab0ddae75..95fb4ed834 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -57,6 +57,11 @@ class LayeredProductionCompileBenchmarkResult(NamedTuple): project_config_edit: CompileBenchmarkMeasurement +class SemanticCompileBenchmarkResult(NamedTuple): + cold: CompileBenchmarkMeasurement + warm: CompileBenchmarkMeasurement + + class DbtShapedCompileBenchmarkResult(NamedTuple): cold_seconds: float warm_median_seconds: float @@ -258,6 +263,50 @@ def run_layered_production_compile_benchmark( ) +def run_semantic_compile_benchmark( + *, + project_dir: Path, + model_count: int, + source_count: int, + seed_count: int, + function_count: int, + macro_count: int, + test_count: int, + audit_count: int, + expected_cold_max_seconds: float, + expected_warm_max_seconds: float, +) -> SemanticCompileBenchmarkResult: + """Measure cold and unchanged compiles for a semantically dense generated project.""" + + skip_actions: dict[bool, Callable[[], None]] = { + False: _continue_compile_benchmark, + True: _skip_compile_benchmark, + } + skip_actions[os.environ.get("SQLBUILD_SKIP_PERFORMANCE_TESTS") == "1"]() + warmup_dir: Path = project_dir.parent / "semantic_compile_runtime_warmup" + write_advanced_compile_project(project_dir=warmup_dir, model_count=32) + _ = _run_compile_benchmark(project_dir=warmup_dir, expected_max_seconds=5.0) + write_semantic_compile_project( + project_dir=project_dir, + model_count=model_count, + source_count=source_count, + seed_count=seed_count, + function_count=function_count, + macro_count=macro_count, + test_count=test_count, + audit_count=audit_count, + ) + cold: CompileBenchmarkMeasurement = _run_profiled_compile_benchmark( + project_dir=project_dir, + expected_max_seconds=expected_cold_max_seconds, + ) + warm: CompileBenchmarkMeasurement = _run_profiled_compile_benchmark( + project_dir=project_dir, + expected_max_seconds=expected_warm_max_seconds, + ) + return SemanticCompileBenchmarkResult(cold=cold, warm=warm) + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", @@ -339,6 +388,13 @@ def measure_compiled_test_sql_bytes(project_dir: Path) -> int: return sum(path.stat().st_size for path in compiled_tests_dir.rglob("*.sql")) +def measure_declared_model_columns(project_dir: Path) -> int: + return sum( + path.read_text(encoding="utf-8").count("(type ") + for path in (project_dir / "models").rglob("*.sql") + ) + + @contextmanager def _fail_after_seconds(seconds: float) -> Iterator[None]: def _raise_timeout(signum: int, frame: FrameType | None) -> None: @@ -679,6 +735,18 @@ def _test_heavy_sql(*, base_index: int, target_index: int, fixture_row_count: in (0.999, 260_000), (1.0, 520_000), ) +_SEMANTIC_COLUMN_COUNT_PROFILE: tuple[tuple[float, int], ...] = ( + (0.0, 3), + (0.25, 9), + (0.50, 19), + (0.75, 33), + (0.90, 58), + (0.95, 115), + (0.99, 232), + (1.0, 538), +) +_SEMANTIC_SET_OPERATION_MODEL_INDEX: int = _SPINE_DEPTH +_SEMANTIC_SET_OPERATION_BRANCH_COUNT: int = 267 def write_layered_production_compile_project( @@ -718,6 +786,43 @@ def write_layered_production_compile_project( ) +def write_semantic_compile_project( + *, + project_dir: Path, + model_count: int, + source_count: int, + seed_count: int, + function_count: int, + macro_count: int, + test_count: int, + audit_count: int, +) -> None: + """Write a neutral project with broad resources and dense SQL semantics.""" + + _layered_write_project_config(project_dir=project_dir) + _layered_write_sources(project_dir=project_dir, source_count=source_count) + _layered_write_seeds(project_dir=project_dir, seed_count=seed_count) + _layered_write_functions(project_dir=project_dir, function_count=function_count) + _layered_write_macros(project_dir=project_dir, macro_count=macro_count) + _layered_write_hooks(project_dir=project_dir) + _semantic_write_models( + project_dir=project_dir, + model_count=model_count, + source_count=source_count, + seed_count=seed_count, + function_count=function_count, + macro_count=macro_count, + audit_count=audit_count, + ) + _layered_write_tests( + project_dir=project_dir, + model_count=model_count, + test_count=test_count, + source_count=source_count, + semantic_fixture_scale=True, + ) + + def _layered_write_project_config(*, project_dir: Path) -> None: project_dir.mkdir(parents=True) (project_dir / "sqlbuild_project.toml").write_text( @@ -1043,8 +1148,196 @@ def _layered_dependent_model_sql( ) +def _semantic_write_models( + *, + project_dir: Path, + model_count: int, + source_count: int, + seed_count: int, + function_count: int, + macro_count: int, + audit_count: int, +) -> None: + for index in range(model_count): + model_dir: Path = project_dir / "models" / _layered_model_folder(index=index) + model_dir.mkdir(parents=True, exist_ok=True) + column_count: int = _semantic_model_column_count(index=index, model_count=model_count) + builders: dict[bool, Callable[[], str]] = { + True: lambda index=index, column_count=column_count: _semantic_set_operation_model_sql( + index=index, + source_count=source_count, + column_count=column_count, + model_count=model_count, + audit_count=audit_count, + ), + False: lambda index=index, column_count=column_count: _semantic_regular_model_sql( + index=index, + model_count=model_count, + source_count=source_count, + seed_count=seed_count, + function_count=function_count, + macro_count=macro_count, + audit_count=audit_count, + column_count=column_count, + ), + } + sql: str = builders[index == _SEMANTIC_SET_OPERATION_MODEL_INDEX]() + (model_dir / f"model_{index:05d}.sql").write_text(sql, encoding="utf-8") + + +def _semantic_model_column_count(*, index: int, model_count: int) -> int: + quantile: float = (index + 1) / model_count + upper_quantiles: tuple[float, ...] = tuple(item[0] for item in _SEMANTIC_COLUMN_COUNT_PROFILE) + upper_index: int = bisect_left(upper_quantiles, quantile) + lower_quantile, lower_count = _SEMANTIC_COLUMN_COUNT_PROFILE[upper_index - 1] + upper_quantile, upper_count = _SEMANTIC_COLUMN_COUNT_PROFILE[upper_index] + position: float = (quantile - lower_quantile) / (upper_quantile - lower_quantile) + profile_count: int = max(3, round(lower_count + position * (upper_count - lower_count))) + return { + False: profile_count, + True: 270, + }[index == _SEMANTIC_SET_OPERATION_MODEL_INDEX] + + +def _semantic_model_header( + *, index: int, model_count: int, audit_count: int, column_count: int +) -> str: + base_audit_count, extra_audit_count = divmod(audit_count, model_count) + model_audit_count: int = base_audit_count + int(index < extra_audit_count) + base_columns: list[tuple[str, str]] = [ + ("id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), + ] + generated_columns: list[tuple[str, str]] = [ + (f"metric_{column_index:04d}", "DOUBLE") + for column_index in range(column_count - len(base_columns)) + ] + columns: list[tuple[str, str]] = [*base_columns, *generated_columns] + declarations: list[str] = [] + for column_index, (name, data_type) in enumerate(columns): + audit_sql: str = {False: "", True: ", audits [not_null]"}[column_index < model_audit_count] + nullable_sql: str = {False: "", True: ", nullable false"}[column_index == 0] + declarations.append(f" {name} (type {data_type}{nullable_sql}{audit_sql})") + return "MODEL (\n columns (\n" + ",\n".join(declarations) + "\n ),\n);" + + +def _semantic_regular_model_sql( + *, + index: int, + model_count: int, + source_count: int, + seed_count: int, + function_count: int, + macro_count: int, + audit_count: int, + column_count: int, +) -> str: + header: str = _semantic_model_header( + index=index, + model_count=model_count, + audit_count=audit_count, + column_count=column_count, + ) + source_index: int = _layered_base_source_index(index=index, source_count=source_count) + relation_sql: str = { + True: f'__source("source_{source_index:05d}")', + False: f'__ref("model_{index - 1:05d}")', + }[_layered_is_base_model(index=index)] + metric_expressions: str = "".join( + f",\n CAST(COALESCE(CASE WHEN id % {column_index % 11 + 2} = 0 " + f"THEN amount + {column_index} WHEN status = 'priority' " + f"THEN amount * {column_index % 7 + 1} ELSE amount - {column_index} END, 0) AS DOUBLE) " + f"AS metric_{column_index:04d}" + for column_index in range(column_count - 3) + ) + seed_index: int = index % seed_count + seed_join_sql: str = { + True: f'\nLEFT JOIN __seed("seed_{seed_index:05d}") AS seed ON seed.id = input.id', + False: "", + }[index >= _SEED_REFERENCE_START_INDEX and index % _SEED_INTERVAL == 0] + function_index: int = index % function_count + amount_expression: str = { + True: f'__udf("fn_{function_index:05d}")(amount)', + False: "amount + CAST(@@benchmark_revision AS INTEGER)", + }[index % _FUNCTION_INTERVAL == 0] + macro_index: int = (index // _MACRO_INTERVAL) % macro_count + id_expression: str = { + True: f'@macro_{macro_index:05d}("id")', + False: "id", + }[index % _MACRO_INTERVAL == 0] + direct_sql: str = f"""SELECT + CAST({id_expression} AS INTEGER) AS id, + CAST({amount_expression} AS DOUBLE) AS amount, + CAST(CASE WHEN id % 2 = 0 THEN 'even' ELSE 'odd' END AS VARCHAR) AS status{metric_expressions} +FROM {relation_sql} AS input{seed_join_sql} +""" + with_sql: str = f"WITH transformed AS (\n{direct_sql.rstrip()}\n)\nSELECT * FROM transformed\n" + nested_sql: str = f"SELECT * FROM (\n{direct_sql.rstrip()}\n) AS nested_query\n" + query_sql: str = { + (True, True): with_sql, + (True, False): with_sql, + (False, True): nested_sql, + (False, False): direct_sql, + }[ + ( + index % _TOP_LEVEL_WITH_INTERVAL == 0, + index % _NESTED_QUERY_INTERVAL == 0, + ) + ] + return f"{header}\n\n{query_sql}" + + +def _semantic_set_operation_model_sql( + *, + index: int, + source_count: int, + column_count: int, + model_count: int, + audit_count: int, +) -> str: + header: str = _semantic_model_header( + index=index, + model_count=model_count, + audit_count=audit_count, + column_count=column_count, + ) + source_index: int = _layered_base_source_index(index=index, source_count=source_count) + branches: str = "\nUNION ALL\n".join( + f"SELECT {branch % 5 + 1} AS bucket, {branch * 10} AS offset_seconds, " + f"CAST({branch % 13} AS DOUBLE) AS adjustment" + for branch in range(_SEMANTIC_SET_OPERATION_BRANCH_COUNT) + ) + metrics: str = "".join( + f",\n CAST(MAX(CASE WHEN bucket = {metric % 5 + 1} " + f"AND offset_seconds = {metric * 10} THEN amount + adjustment END) AS DOUBLE) " + f"AS metric_{metric:04d}" + for metric in range(column_count - 3) + ) + return f"""{header} + +WITH offset_grid AS ( +{branches} +), measurements AS ( + SELECT source.id, source.amount, source.status, grid.bucket, grid.offset_seconds, grid.adjustment + FROM __source("source_{source_index:05d}") AS source + CROSS JOIN offset_grid AS grid +) +SELECT + CAST(MAX(id) AS INTEGER) AS id, + CAST(MAX(amount) AS DOUBLE) AS amount, + CAST(MAX(status) AS VARCHAR) AS status{metrics} +FROM measurements +""" + + def _layered_write_tests( - *, project_dir: Path, model_count: int, test_count: int, source_count: int + *, + project_dir: Path, + model_count: int, + test_count: int, + source_count: int, + semantic_fixture_scale: bool = False, ) -> None: tests_dir: Path = project_dir / "tests" / "unit" tests_dir.mkdir(parents=True) @@ -1059,6 +1352,7 @@ def _layered_write_tests( model_count=model_count, source_count=source_count, repeated_target_count=repeated_target_count, + semantic_fixture_scale=semantic_fixture_scale, ) for test_index in range( first_test_index, @@ -1074,14 +1368,21 @@ def _layered_test_block( model_count: int, source_count: int, repeated_target_count: int, + semantic_fixture_scale: bool = False, ) -> str: group_count: int = (model_count - _SPINE_DEPTH) // _TEST_CHAIN_DEPTH - representative_group_count: int = (group_count * 2) // 3 + representative_group_count: int = { + True: group_count // 3, + False: (group_count * 2) // 3, + }[semantic_fixture_scale] group_index: int = ((test_index // 5) * representative_group_count) // repeated_target_count base_index: int = _SPINE_DEPTH + group_index * _TEST_CHAIN_DEPTH target_index: int = base_index + _TEST_CHAIN_DEPTH - 1 source_index: int = _layered_base_source_index(index=base_index, source_count=source_count) - fixture_row_count: int = 40 + (test_index % 5) * 40 + fixture_row_count: int = { + True: 5 + (test_index % 5) * 5, + False: 40 + (test_index % 5) * 40, + }[semantic_fixture_scale] return _layered_test_sql( test_index=test_index, source_index=source_index, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_performance.py index 0bad38e41c..e8de4d5950 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_performance.py @@ -12,17 +12,21 @@ CompileScalingGuardTestCase, DbtShapedCompilePerformanceGuardTestCase, LayeredProductionCompilePerformanceGuardTestCase, + SemanticCompilePerformanceGuardTestCase, SqlTestHeavyCompilePerformanceGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( CompileBenchmarkMeasurement, DbtShapedCompileBenchmarkResult, LayeredProductionCompileBenchmarkResult, + SemanticCompileBenchmarkResult, measure_compiled_test_sql_bytes, + measure_declared_model_columns, measure_model_sql_bytes, run_advanced_compile_benchmark, run_dbt_shaped_compile_benchmark, run_layered_production_compile_benchmark, + run_semantic_compile_benchmark, run_test_heavy_compile_benchmark, ) @@ -278,8 +282,7 @@ def test_given_layered_production_project_when_compiling_and_editing_then_report f"{phase}={milliseconds}ms" for phase, milliseconds in measurement.timings_ms.items() ) _LOGGER.info( - f"dagster-shaped compile path={label} total={measurement.elapsed_seconds:.3f}s " - f"{phase_text}" + f"layered compile path={label} total={measurement.elapsed_seconds:.3f}s {phase_text}" ) assert result.cold.summary == { @@ -310,3 +313,81 @@ def test_given_layered_production_project_when_compiling_and_editing_then_report assert result.test_edit.elapsed_seconds < test_case.expected_edit_max_seconds assert result.macro_edit.elapsed_seconds < test_case.expected_edit_max_seconds assert result.project_config_edit.elapsed_seconds < test_case.expected_config_edit_max_seconds + + +@pytest.mark.performance +@pytest.mark.parametrize( + "test_case", + [ + SemanticCompilePerformanceGuardTestCase( + description="Semantically dense compile stays within cold and warm budgets", + model_count=976, + source_count=232, + seed_count=46, + function_count=23, + macro_count=12, + test_count=958, + audit_count=1_645, + expected_min_declared_columns=30_000, + expected_max_declared_columns=38_000, + expected_min_model_sql_bytes=5_500_000, + expected_max_model_sql_bytes=7_500_000, + expected_min_compiled_test_bytes=10_000_000, + expected_max_compiled_test_bytes=20_000_000, + expected_cold_max_seconds=60.0, + expected_warm_max_seconds=12.0, + ) + ], + ids=lambda case: case.description, +) +def test_given_semantically_dense_project_when_compiling_then_reports_phased_budgets( + tmp_path: Path, + test_case: SemanticCompilePerformanceGuardTestCase, +) -> None: + project_dir: Path = tmp_path / "semantic_compile" + result: SemanticCompileBenchmarkResult = run_semantic_compile_benchmark( + project_dir=project_dir, + model_count=test_case.model_count, + source_count=test_case.source_count, + seed_count=test_case.seed_count, + function_count=test_case.function_count, + macro_count=test_case.macro_count, + test_count=test_case.test_count, + audit_count=test_case.audit_count, + expected_cold_max_seconds=test_case.expected_cold_max_seconds, + expected_warm_max_seconds=test_case.expected_warm_max_seconds, + ) + for label, measurement in {"cold": result.cold, "warm": result.warm}.items(): + phase_text: str = " ".join( + f"{phase}={milliseconds}ms" for phase, milliseconds in measurement.timings_ms.items() + ) + _LOGGER.info( + f"semantic compile path={label} total={measurement.elapsed_seconds:.3f}s {phase_text}" + ) + + assert result.cold.summary == { + "models": test_case.model_count, + "selected_models": test_case.model_count, + "sources": test_case.source_count, + "seeds": test_case.seed_count, + "selected_seeds": test_case.seed_count, + "functions": test_case.function_count, + "selected_functions": test_case.function_count, + "audits": test_case.audit_count, + "tests": test_case.test_count, + "hooks": 2, + "execution_layers": 54, + "errors": 0, + "warnings": 0, + } + declared_columns: int = measure_declared_model_columns(project_dir) + model_sql_bytes: int = measure_model_sql_bytes(project_dir) + compiled_test_bytes: int = measure_compiled_test_sql_bytes(project_dir) + assert test_case.expected_min_declared_columns <= declared_columns + assert declared_columns <= test_case.expected_max_declared_columns + assert test_case.expected_min_model_sql_bytes <= model_sql_bytes + assert model_sql_bytes <= test_case.expected_max_model_sql_bytes + assert test_case.expected_min_compiled_test_bytes <= compiled_test_bytes + assert compiled_test_bytes <= test_case.expected_max_compiled_test_bytes + assert result.cold.elapsed_seconds < test_case.expected_cold_max_seconds + assert result.warm.elapsed_seconds < test_case.expected_warm_max_seconds