diff --git a/src/sqlbuild/lint/_helpers/project_files.py b/src/sqlbuild/lint/_helpers/project_files.py index dcbd98fba4..01e0f21c60 100644 --- a/src/sqlbuild/lint/_helpers/project_files.py +++ b/src/sqlbuild/lint/_helpers/project_files.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES @@ -12,8 +13,9 @@ def collect_project_files( project_dir: Path, selected_paths: frozenset[Path] | None = None, source_files: dict[Path, str] | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, ) -> dict[Path, str]: - """Return authored SQL files keyed by path, reusing contents a caller already read.""" + """Return authored SQL files keyed by path, sharing identical text discovery already holds.""" if source_files is not None: return { @@ -21,6 +23,9 @@ def collect_project_files( for file_path, contents in source_files.items() if selected_paths is None or file_path.resolve() in selected_paths } + discovered: dict[Path, str] = ( + {} if discovered_inputs is None else discovered_sql_contents(discovered_inputs) + ) files: dict[Path, str] = {} directory_name: str for directory_name in LINT_DIRECTORY_NAMES: @@ -32,10 +37,31 @@ def collect_project_files( if selected_paths is not None and file_path.resolve() not in selected_paths: continue with file_path.open("r", encoding="utf-8", newline="") as handle: - files[file_path] = handle.read() + contents: str = handle.read() + known: str | None = discovered.get(file_path) + files[file_path] = known if known is not None and known == contents else contents return files +def discovered_sql_contents(discovered_inputs: DiscoveredProjectInputs) -> dict[Path, str]: + """Return the text discovery read for every SQL file it retains, keyed by path.""" + + contents: dict[Path, str] = {} + for group in ( + discovered_inputs.model_files, + discovered_inputs.enum_files, + discovered_inputs.constant_files, + discovered_inputs.model_schema_files, + discovered_inputs.sql_function_files, + discovered_inputs.sql_hook_files, + discovered_inputs.test_files, + discovered_inputs.scenario_files, + discovered_inputs.audit_files, + ): + contents.update({item.file_path: item.contents for item in group}) + return contents + + def sort_violations(violations: list) -> tuple: """Return violations sorted by file path then position.""" diff --git a/src/sqlbuild/lint/main/collect_project_files.py b/src/sqlbuild/lint/main/collect_project_files.py index 0a7d59e43c..dde2fd282e 100644 --- a/src/sqlbuild/lint/main/collect_project_files.py +++ b/src/sqlbuild/lint/main/collect_project_files.py @@ -2,12 +2,20 @@ from pathlib import Path +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.lint._helpers.project_files import collect_project_files as _collect_project_files def collect_project_files( - *, project_dir: Path, selected_paths: frozenset[Path] | None + *, + project_dir: Path, + selected_paths: frozenset[Path] | None, + discovered_inputs: DiscoveredProjectInputs | None = None, ) -> dict[Path, str]: - """Collect supported SQL-analysis inputs for a project or selected path set.""" + """Collect supported SQL-analysis inputs, sharing text discovery already holds.""" - return _collect_project_files(project_dir=project_dir, selected_paths=selected_paths) + return _collect_project_files( + project_dir=project_dir, + selected_paths=selected_paths, + discovered_inputs=discovered_inputs, + ) diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index 9756337ee1..768dd20d0b 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -50,7 +50,10 @@ def run_lint( """Lint all DSL files without modifying anything; optionally skip unexpandable SQL.""" files: dict[Path, str] = collect_project_files( - project_dir=project_dir, selected_paths=selected_paths, source_files=source_files + project_dir=project_dir, + selected_paths=selected_paths, + source_files=source_files, + discovered_inputs=discovered_inputs, ) violations: list[LintViolation] = [] context: SqlExpansionContext | None = expansion_context or _expansion_context( diff --git a/src/sqlbuild/rule_engine/_helpers/run/memory.py b/src/sqlbuild/rule_engine/_helpers/run/memory.py new file mode 100644 index 0000000000..ddc6c45869 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/memory.py @@ -0,0 +1,30 @@ +"""Return memory the C allocator keeps after earlier compile phases freed it.""" + +from __future__ import annotations + +import ctypes +import sys +from collections.abc import Callable +from functools import cache +from typing import Any + + +def release_freed_memory() -> None: + """Hand freed glibc heap pages back to the OS; a no-op on other platforms and allocators.""" + + trim: Callable[[int], int] | None = _malloc_trim() + if trim is not None: + _ = trim(0) + + +@cache +def _malloc_trim() -> Callable[[int], int] | None: + if not sys.platform.startswith("linux"): + return None + try: + trim: Any = ctypes.CDLL(None).malloc_trim + except (OSError, AttributeError): + return None + trim.argtypes = (ctypes.c_size_t,) + trim.restype = ctypes.c_int + return trim diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index a4053439f7..d537b4740d 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -60,6 +60,7 @@ ) from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints +from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config @@ -106,6 +107,7 @@ def evaluate_rules( no_sql_analysis: bool = False, ) -> RulesRunResult: """Evaluate independent rule phases concurrently, then finalize their combined findings.""" + release_freed_memory() effective_config: RulesConfig = resolve_rule_ignore_selectors( config=config, project=graph.project ) @@ -185,6 +187,7 @@ def evaluate_rules( dialect=_lint_identity(lint_config), project_dir=resolved_project_dir, collect_files=prepared_sql is None and model_paths is None, + discovered_inputs=discovered_inputs, ) sql_started: float = time.monotonic() sql_result: _SqlRulesEvaluation = _run_sql_rules( @@ -239,6 +242,7 @@ def _prepare_sql_rule_inputs( dialect: str, project_dir: Path, collect_files: bool, + discovered_inputs: DiscoveredProjectInputs, ) -> _SqlRuleInputs: """Hash SQL and read the project files on the caller while built-in rules evaluate natively.""" @@ -248,7 +252,9 @@ def _prepare_sql_rule_inputs( return _SqlRuleInputs( identities=identities, project_files=( - collect_project_files(project_dir=project_dir, selected_paths=None) + collect_project_files( + project_dir=project_dir, selected_paths=None, discovered_inputs=discovered_inputs + ) if identities and collect_files else None ), @@ -529,7 +535,11 @@ def _run_sql_rules( project_files = ( prepared_inputs.project_files if prepared_inputs is not None and prepared_inputs.project_files is not None - else collect_project_files(project_dir=project_dir, selected_paths=None) + else collect_project_files( + project_dir=project_dir, + selected_paths=None, + discovered_inputs=discovered_inputs, + ) ) file_path: Path contents: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index dc07122711..5179524f91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -62,7 +62,7 @@ "dense_models_10000_all_rules", 10000, 145.0, - 7 * _GIB // 2, + 13 * _GIB // 4, "913a08c2277304efacf75e293728767a3fa416f6b32f5488dca6ab5baea80668", ), ), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index c8c4cb3bc5..743c8ed0cf 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -909,10 +909,17 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ original_collect: Callable[..., dict[Path, str]] = rules_module.collect_project_files def counting_collect( - *, project_dir: Path, selected_paths: frozenset[Path] | None + *, + project_dir: Path, + selected_paths: frozenset[Path] | None, + discovered_inputs: DiscoveredProjectInputs | None = None, ) -> dict[Path, str]: collections.append(selected_paths) - return original_collect(project_dir=project_dir, selected_paths=selected_paths) + return original_collect( + project_dir=project_dir, + selected_paths=selected_paths, + discovered_inputs=discovered_inputs, + ) monkeypatch.setattr(rules_module, "collect_project_files", counting_collect) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index b49d2a90bb..ac1722759e 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -234,3 +234,12 @@ class BatchedPreparationFailureTestCase: unexpandable_file: str skip_unexpandable: bool expected_error: type[Exception] | None + + +@dataclass(frozen=True) +class CollectProjectFilesTestCase: + """Test case for sharing discovered SQL text when collecting lint inputs.""" + + description: str + model_bytes: bytes + expected_shared: bool diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index cf3135758f..e46fed4c20 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -91,3 +91,24 @@ def refuse_preparation(request: NativeLintPreparationRequest) -> NativePreparedS """Fail the per-body native preparation of any body.""" raise ValueError(f"cannot prepare {request['dialect']} body") + + +def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path: + """Write a DuckDB project with one model and one SQL test; return the model path.""" + + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n', encoding="utf-8" + ) + model_path: Path = project_dir / "models" / "orders.sql" + model_path.parent.mkdir(parents=True) + _ = model_path.write_bytes(model_bytes) + test_path: Path = project_dir / "tests" / "test_orders.sql" + test_path.parent.mkdir(parents=True) + _ = test_path.write_text( + "TEST (name orders_keep_ids);\n" + "WITH orders AS (SELECT 1 AS order_id),\n" + "expected AS (SELECT 1 AS order_id)\n" + "SELECT * FROM orders\n", + encoding="utf-8", + ) + return model_path diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py new file mode 100644 index 0000000000..693e961d64 --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -0,0 +1,58 @@ +"""Unit tests for collecting authored SQL files for lint runs.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.lint._helpers.project_files import collect_project_files +from sqlbuild.lint.main.run_lint import run_lint +from sqlbuild.lint.models import LintConfig, LintRunResult +from tests.unit.src.sqlbuild.lint._helpers._test_types import CollectProjectFilesTestCase +from tests.unit.src.sqlbuild.lint._helpers.helpers import write_orders_lint_project + + +@pytest.mark.parametrize( + "test_case", + [ + CollectProjectFilesTestCase( + description="identical text shares the discovered string", + model_bytes=b"MODEL (materialized table);\nSELECT 1 AS order_id\n", + expected_shared=True, + ), + CollectProjectFilesTestCase( + description="carriage returns keep the exact authored text", + model_bytes=b"MODEL (materialized table);\r\nSELECT 1 AS order_id\r\n", + expected_shared=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_discovered_project_when_collecting_files_then_returns_exact_authored_text( + test_case: CollectProjectFilesTestCase, tmp_path: Path +) -> None: + model_path: Path = write_orders_lint_project( + project_dir=tmp_path, model_bytes=test_case.model_bytes + ) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + discovered: dict[Path, str] = {item.file_path: item.contents for item in inputs.model_files} + + files: dict[Path, str] = collect_project_files(project_dir=tmp_path, discovered_inputs=inputs) + shared: LintRunResult = run_lint( + project_dir=tmp_path, config=LintConfig(), discovered_inputs=inputs + ) + fresh: LintRunResult = run_lint(project_dir=tmp_path, config=LintConfig()) + + assert files == collect_project_files(project_dir=tmp_path) + assert files[model_path] == test_case.model_bytes.decode("utf-8") + assert (files[model_path] is discovered[model_path]) is test_case.expected_shared + assert shared.source_texts == fresh.source_texts + assert shared.violations == fresh.violations + assert shared.files_checked == fresh.files_checked == 2 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"]))