From ca84d84d3668fc182293063df731988080048d43 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:06:24 +0100 Subject: [PATCH 1/3] perf(native): share equal inferred columns across session analyses and guard custom rule CPU Native session analyses built a new InferredColumn per model column, while Python's compact analysis shares one object per distinct column. Custom-rule hosts therefore pickled and decoded more distinct objects under native-preview. Adds custom_rules_cpu_ms (rule thread plus host children CPU) and a same-runner ratio entry for it. --- .github/workflows/verify.yml | 6 +++++ .../compile_performance_ratio/constants.py | 1 + .../cli/commands/_helpers/compile/pipeline.py | 1 + .../cli/commands/main/project/_compile.py | 1 + src/sqlbuild/cli/compile/models.py | 1 + .../analysis_session/_helpers/session_rows.py | 24 +++++++++++++++---- .../classes/native_model_analysis.py | 7 ++++-- .../_helpers/engine/custom_rules.py | 14 +++++++++++ .../rule_engine/_helpers/engine/native.py | 1 + .../rule_engine/_helpers/run/rules.py | 1 + src/sqlbuild/rule_engine/models.py | 3 +++ .../test_compare_compile_performance.py | 12 ++++++++++ .../main/build/test_compile_json_behavior.py | 1 + .../compile/test_dense_compile_performance.py | 1 + .../compiler/analysis_session/_test_types.py | 1 + .../compiler/analysis_session/helpers.py | 12 ++++++++++ .../test_native_model_analysis.py | 6 +++++ 17 files changed, 87 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 84833fa759..618f4e5128 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -611,6 +611,12 @@ jobs: gate_phase: model_analysis_cpu_ms compile_args: --compile-arg=--select=* max_ratio: "1.50" + - kind: dense + label: native-preview custom rules vs python + base_engine: python + head_engine: native-preview + gate_phase: custom_rules_cpu_ms + max_ratio: "1.05" steps: - uses: actions/checkout@v4 - name: Check out the comparison base diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 0a365a0367..2c061917a9 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -53,6 +53,7 @@ "contracts_cpu_ms", "built_in_rules_ms", "custom_rules_ms", + "custom_rules_cpu_ms", "test_planning_ms", "write_ms", ) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 5ecad25968..c30b70555b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -260,6 +260,7 @@ def _analyze_compile_project( contract_cpu_ms=contract_cpu_ms, built_in_rules_ms=rules_result.built_in_ms, custom_rules_ms=rules_result.custom_ms, + custom_rules_cpu_ms=rules_result.custom_cpu_ms, early_lint_wait_ms=early_lint_wait_ms, rule_cache_hits=rules_result.cache_hits, rule_cache_misses=rules_result.cache_misses, diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 6ea2d58d06..6552c8e429 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -169,6 +169,7 @@ def _run_compile_with_status( "contracts_cpu_ms": analysis.contract_cpu_ms, "built_in_rules_ms": analysis.built_in_rules_ms, "custom_rules_ms": analysis.custom_rules_ms, + "custom_rules_cpu_ms": analysis.custom_rules_cpu_ms, "early_lint_wait_ms": analysis.early_lint_wait_ms, "rule_cache_hits": analysis.rule_cache_hits, "rule_cache_misses": analysis.rule_cache_misses, diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index 1a166ca13a..0f36a5373d 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -45,6 +45,7 @@ class CompileAnalysis: contract_cpu_ms: int built_in_rules_ms: int = 0 custom_rules_ms: int = 0 + custom_rules_cpu_ms: int = 0 early_lint_wait_ms: int = 0 rule_cache_hits: int = 0 rule_cache_misses: int = 0 diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py index 093c361e6e..7a67564147 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py @@ -134,12 +134,28 @@ def deferred_row(*, model: int, analysis: PolyglotAnalysisResult) -> DeferredRow def inferred_columns(rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: + return shared_inferred_columns(rows=rows, shared={}) + + +def shared_inferred_columns( + *, rows: list[ColumnRow] | None, shared: dict[ColumnRow, InferredColumn] +) -> tuple[InferredColumn, ...] | None: + """Columns as Python's analysis shares them: one object per distinct column value.""" + if rows is None: return None - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in rows - ) + return tuple(_shared_column(row=row, shared=shared) for row in rows) + + +def _shared_column(*, row: ColumnRow, shared: dict[ColumnRow, InferredColumn]) -> InferredColumn: + column: InferredColumn | None = shared.get(row) + if column is None: + name, data_type, nullability = row + column = InferredColumn( + name=name, type=data_type, nullability=InferredNullability(nullability) + ) + shared[row] = column + return column def binding_diagnostics(rows: list[DiagnosticRow]) -> tuple[SqlBindingDiagnostic, ...]: diff --git a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py index 8fa6049f45..8d37ffca70 100644 --- a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py @@ -14,8 +14,8 @@ compact_lineage, contract_proof, deferred_row, - inferred_columns, lineage_facts, + shared_inferred_columns, ) from sqlbuild.compiler.analysis_session.constants import ( CONTRACT_DEFERRED, @@ -31,6 +31,7 @@ ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.analysis_session.types import ( + ColumnRow, DeferralRow, DeferredRow, FinishRow, @@ -42,6 +43,7 @@ from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, + InferredColumn, ModelSqlAnalysis, ModelSqlAnalysisRequest, PolyglotAnalysisResult, @@ -69,6 +71,7 @@ def __init__( request.column_nullability_by_table ) self._kept: dict[tuple[int, str], PolyglotAnalysisResult] = {} + self._columns: dict[ColumnRow, InferredColumn] = {} self._deferrals: Counter[str] = Counter() def analyses( @@ -205,7 +208,7 @@ def _model_analysis( return ModelSqlAnalysis( polyglot_analysis=PolyglotAnalysisResult( analysis_succeeded=succeeded, - columns=inferred_columns(columns), + columns=shared_inferred_columns(rows=columns, shared=self._columns), lineage_columns=lineage, has_star=has_star, star_resolved=star_resolved, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index b806fb398c..ccd1fe39d2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -80,6 +80,7 @@ def evaluate_custom_rules_cached( """Reuse results whose recorded fact reads are unchanged and evaluate only the rest.""" started: float = time.monotonic() + started_cpu: float = _cpu_seconds() custom: tuple[Rule, ...] = tuple(sorted(rules, key=lambda rule: rule.code)) if not custom: return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) @@ -105,6 +106,7 @@ def evaluate_custom_rules_cached( cache_hits=0, cache_misses=len(custom), custom_ms=_elapsed_ms(started), + custom_cpu_ms=_cpu_ms(started_cpu), ) digests: FactDigests = FactDigests( views=_views_factory(project=facts_project, project_dir=project_dir, dialect=dialect) @@ -185,6 +187,7 @@ def evaluate_custom_rules_cached( cache_hits=hits, cache_misses=misses, custom_ms=_elapsed_ms(started), + custom_cpu_ms=_cpu_ms(started_cpu), ) @@ -476,5 +479,16 @@ def _digest(value: object) -> str: return hashlib.sha256(orjson.dumps(value, default=str)).hexdigest() +def _cpu_seconds() -> float: + """This thread's CPU plus that of finished child processes, which run the rule hosts.""" + + times: os.times_result = os.times() + return time.thread_time() + times.children_user + times.children_system + + +def _cpu_ms(started: float) -> int: + return int((_cpu_seconds() - started) * 1000) + + def _elapsed_ms(started: float) -> int: return round((time.monotonic() - started) * 1000) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index d9cd1a569b..a28072f54c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -161,6 +161,7 @@ def evaluate_native( cache_misses=(0 if reused else native_misses) + custom.cache_misses, built_in_ms=0 if reused else int(payload.get("built_in_ms", 0)), custom_ms=custom.custom_ms, + custom_cpu_ms=custom.custom_cpu_ms, ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 9f5b2c8c32..6b4ccd8567 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -231,6 +231,7 @@ def evaluate_rules( unevaluated_resources=len({item.path for item in findings if item.unevaluated}), built_in_ms=sql_ms + result.built_in_ms, custom_ms=result.custom_ms, + custom_cpu_ms=result.custom_cpu_ms, cache_hits=result.cache_hits + sql_result.cache_hits, cache_misses=result.cache_misses + sql_result.cache_misses, skipped_type_proof_rules=skipped_type_proof_rules, diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 0915f9ccfb..98831bae24 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -311,6 +311,7 @@ class RulesResult: cache_misses: int = 0 built_in_ms: int = 0 custom_ms: int = 0 + custom_cpu_ms: int = 0 @dataclass(frozen=True) @@ -346,6 +347,7 @@ class RulesRunResult: evaluated_models: int built_in_ms: int custom_ms: int + custom_cpu_ms: int = 0 unevaluated_resources: int = 0 cache_hits: int = 0 cache_misses: int = 0 @@ -431,6 +433,7 @@ class CustomRulesOutcome: cache_hits: int cache_misses: int custom_ms: int + custom_cpu_ms: int = 0 @dataclass(frozen=True) diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py index e85352ce58..827a663c6c 100644 --- a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -206,6 +206,18 @@ def test_given_base_root_when_comparing_compile_performance_then_each_side_uses_ gate_phase="model_analysis_cpu_ms", compile_args=("--compile-arg=--select=*",), ), + EnginePhaseRatioTestCase( + description="custom rule CPU over its allowance fails on it alone", + max_ratio="0", + phase_noise_floor_ms="-1", + expected_return_code=1, + expected_fragments=( + "Gated phases: custom_rules_cpu_ms.", + "| custom_rules_cpu_ms |", + "Compile performance regression: dense 20 cold custom_rules_cpu_ms:", + ), + gate_phase="custom_rules_cpu_ms", + ), ), ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index c7b7680d7c..cbdbaa803f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -71,6 +71,7 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "attachment_ms", "built_in_rules_ms", "custom_rules_ms", + "custom_rules_cpu_ms", "early_lint_wait_ms", "model_analysis_ms", "model_analysis_cpu_ms", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 606879ed41..dec0bf7ea8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -125,6 +125,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert timings["rule_cache_misses"] == 2 * test_case.model_count + 1 assert timings["built_in_rules_ms"] > 0 assert timings["custom_rules_ms"] > 0 + assert timings["custom_rules_cpu_ms"] > 0 assert result.semantic_fingerprint == test_case.expected_fingerprint assert result.elapsed_seconds < test_case.expected_max_wall_seconds assert result.peak_rss_bytes < test_case.expected_max_rss_bytes diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 9ce474e84b..42890b9cfb 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -63,3 +63,4 @@ class SharedAnalysisTestCase: expected_shared: int expected_reanalysed: int expected_unshared: int + expected_column_values: int diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 8115e45b60..843644cb7a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -40,6 +40,7 @@ CompiledLineageColumnFact, CompileProjectInputs, DynamicColumnContractProof, + InferredColumn, ModelSqlAnalysis, ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs @@ -430,6 +431,8 @@ class AnalysisParity: standalone_proofs: int = 0 proven_pivots: int = 0 native_enrichments: int = 0 + native_column_objects: int = 0 + native_column_values: int = 0 def compare_analyses( @@ -459,6 +462,11 @@ def models_by_both( ) ) python: dict[str, ModelSqlAnalysis] = python_analysis.func(**keywords) + columns: list[InferredColumn] = list( + chain.from_iterable(map(_native_columns, (native or {}).values())) + ) + parity.native_column_objects += len(set(map(id, columns))) + parity.native_column_values += len(set(columns)) parity.analysed_models += len(python) _append(parity, "model analyses", _analysis_views(python), _analysis_views(native)) _append( @@ -569,6 +577,10 @@ def deferral_kinds(record_dir: Path) -> Counter[str]: return Counter(f"{record['site']}:{record['kind']}" for record in records) +def _native_columns(analysis: ModelSqlAnalysis) -> tuple[InferredColumn, ...]: + return analysis.polyglot_analysis.columns or () + + def _append(parity: AnalysisParity, name: str, python: object, native: object) -> None: parity.names.append(name) parity.python.append(python) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 34e3dfddb0..39642d496e 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -107,6 +107,7 @@ def test_given_generated_projects_when_analysing_natively_then_matches_python( <= test_case.expected_maximum_enrichment_deferrals ) assert parity.native_enrichments >= test_case.expected_minimum_native_enrichments + assert parity.native_column_objects == parity.native_column_values > 0 @pytest.mark.parametrize( @@ -121,6 +122,7 @@ def test_given_generated_projects_when_analysing_natively_then_matches_python( expected_shared=22, expected_reanalysed=6, expected_unshared=2, + expected_column_values=12, ) ], ids=lambda case: case.description, @@ -153,12 +155,16 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_match sum(session.sharing[1] for session in sessions), parity.analysed_models - shared, deferral_kinds(record_dir), + parity.native_column_objects, + parity.native_column_values, ) == ( test_case.expected_analysed, test_case.expected_shared, test_case.expected_reanalysed, test_case.expected_unshared, Counter(), + test_case.expected_column_values, + test_case.expected_column_values, ) From 10fc7719b5d23d91f20769e6dab84dcfab75ffa9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:14:59 +0100 Subject: [PATCH 2/3] refactor(native): keep the shared column memo on the session analysis --- .../analysis_session/_helpers/session_rows.py | 24 ++++--------------- .../classes/native_model_analysis.py | 20 ++++++++++++++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py index 7a67564147..093c361e6e 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py @@ -134,28 +134,12 @@ def deferred_row(*, model: int, analysis: PolyglotAnalysisResult) -> DeferredRow def inferred_columns(rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: - return shared_inferred_columns(rows=rows, shared={}) - - -def shared_inferred_columns( - *, rows: list[ColumnRow] | None, shared: dict[ColumnRow, InferredColumn] -) -> tuple[InferredColumn, ...] | None: - """Columns as Python's analysis shares them: one object per distinct column value.""" - if rows is None: return None - return tuple(_shared_column(row=row, shared=shared) for row in rows) - - -def _shared_column(*, row: ColumnRow, shared: dict[ColumnRow, InferredColumn]) -> InferredColumn: - column: InferredColumn | None = shared.get(row) - if column is None: - name, data_type, nullability = row - column = InferredColumn( - name=name, type=data_type, nullability=InferredNullability(nullability) - ) - shared[row] = column - return column + return tuple( + InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) + for name, data_type, nullability in rows + ) def binding_diagnostics(rows: list[DiagnosticRow]) -> tuple[SqlBindingDiagnostic, ...]: diff --git a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py index 8d37ffca70..86dd27bf9e 100644 --- a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py @@ -15,7 +15,6 @@ contract_proof, deferred_row, lineage_facts, - shared_inferred_columns, ) from sqlbuild.compiler.analysis_session.constants import ( CONTRACT_DEFERRED, @@ -179,6 +178,23 @@ def _record_catalog_changes( }, ) + def _inferred_columns(self, rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: + """Columns as Python's analysis shares them: one object per distinct column value.""" + + if rows is None: + return None + return tuple(map(self._inferred_column, rows)) + + def _inferred_column(self, row: ColumnRow) -> InferredColumn: + column: InferredColumn | None = self._columns.get(row) + if column is None: + name, data_type, nullability = row + column = InferredColumn( + name=name, type=data_type, nullability=InferredNullability(nullability) + ) + self._columns[row] = column + return column + def _model_analysis( self, *, @@ -208,7 +224,7 @@ def _model_analysis( return ModelSqlAnalysis( polyglot_analysis=PolyglotAnalysisResult( analysis_succeeded=succeeded, - columns=shared_inferred_columns(rows=columns, shared=self._columns), + columns=self._inferred_columns(columns), lineage_columns=lineage, has_star=has_star, star_resolved=star_resolved, From bd20f844a5b39af02735e2cc85f6cea51bc11e18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:21:22 +0100 Subject: [PATCH 3/3] ci: gate native-preview custom rules CPU only against gross regressions --- .github/workflows/verify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 618f4e5128..85713175bf 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -616,7 +616,7 @@ jobs: base_engine: python head_engine: native-preview gate_phase: custom_rules_cpu_ms - max_ratio: "1.05" + max_ratio: "1.25" steps: - uses: actions/checkout@v4 - name: Check out the comparison base