diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 78d512d627..369e300c65 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -4,7 +4,10 @@ from pathlib import Path -from sqlbuild.compiler.compile._helpers.render.macros import find_macro_call_names +from sqlbuild.compiler.compile._helpers.render.macros import ( + find_macro_call_names, + find_nested_macro_call_names, +) from sqlbuild.compiler.compile._helpers.scenarios.core import ( extract_sql_scenario_expected_model_names, ) @@ -30,12 +33,15 @@ DeclarationRecord, GrantRecord, ResourceIdentity, + ResourceRecord, ScopeIndex, ScopeLookup, VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind +type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] + def build_scope_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex @@ -43,11 +49,16 @@ def build_scope_relationship_grants( """Return expected-model grants while retaining independent extraction faults.""" lookup: ScopeLookup = build_scope_lookup(index=index) + shared_declarations: _SharedDeclarations = _shared_declarations_by_directory(lookup=lookup) test_grants, test_faults = _test_relationship_grants( - discovered_inputs=discovered_inputs, lookup=lookup + discovered_inputs=discovered_inputs, + lookup=lookup, + shared_declarations=shared_declarations, ) scenario_grants, scenario_faults = _scenario_relationship_grants( - discovered_inputs=discovered_inputs, lookup=lookup + discovered_inputs=discovered_inputs, + lookup=lookup, + shared_declarations=shared_declarations, ) return ScopeRelationshipBuild( grants=tuple(dict.fromkeys((*test_grants, *scenario_grants))), @@ -56,7 +67,10 @@ def build_scope_relationship_grants( def _test_relationship_grants( - *, discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup + *, + discovered_inputs: DiscoveredProjectInputs, + lookup: ScopeLookup, + shared_declarations: _SharedDeclarations, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] @@ -75,7 +89,8 @@ def _test_relationship_grants( ResourceKind.TEST, block.name or test_file.relative_path.stem ), expected_model_names=expected_names, - include_macros=True, + shared_declarations=shared_declarations, + called_macros=frozenset(find_nested_macro_call_names(block.sql_body)), ) ) if block.mode is SqlTestMode.MACRO: @@ -95,7 +110,10 @@ def _test_relationship_grants( def _scenario_relationship_grants( - *, discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup + *, + discovered_inputs: DiscoveredProjectInputs, + lookup: ScopeLookup, + shared_declarations: _SharedDeclarations, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] @@ -109,6 +127,7 @@ def _scenario_relationship_grants( lookup=lookup, resource=ResourceIdentity(ResourceKind.SCENARIO, scenario.name), expected_model_names=expected_names, + shared_declarations=shared_declarations, ) ) except Exception as error: @@ -121,21 +140,18 @@ def _expected_model_grants( lookup: ScopeLookup, resource: ResourceIdentity, expected_model_names: tuple[str, ...], - include_macros: bool = False, + shared_declarations: _SharedDeclarations, + called_macros: frozenset[str] = frozenset(), ) -> list[GrantRecord]: grants: list[GrantRecord] = [] for model_name in expected_model_names: through: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - resolution: VisibilityResolution = resolve_scope_visibility(lookup=lookup, target=through) - for visible in resolution.visible: - records: tuple[DeclarationRecord, ...] = lookup.declarations.get( - visible.declaration, () - ) - if not records: - continue - declaration: DeclarationRecord = records[0] - if declaration.scope is ScopeKind.PRIVATE or ( - declaration.identity.kind is DeclarationKind.MACRO and not include_macros + for declaration in _shared_visible_declarations( + lookup=lookup, resource=through, shared=shared_declarations + ): + if ( + declaration.identity.kind is DeclarationKind.MACRO + and declaration.identity.name not in called_macros ): continue grants.append( @@ -148,6 +164,54 @@ def _expected_model_grants( return grants +def _shared_declarations_by_directory(*, lookup: ScopeLookup) -> _SharedDeclarations: + """Resolve shared non-private visibility once per model directory and ownership root.""" + + representatives: dict[tuple[str, str], ResourceIdentity] = {} + for identity, records in lookup.resources.items(): + if ( + identity.kind is ResourceKind.MODEL + and len(records) == 1 + and identity not in lookup.grants_by_resource + ): + representatives.setdefault(_directory_key(record=records[0]), identity) + return { + key: _resolve_shared_declarations(lookup=lookup, resource=identity) + for key, identity in representatives.items() + } + + +def _directory_key(*, record: ResourceRecord) -> tuple[str, str]: + return (Path(record.path).parent.as_posix(), record.ownership_root.path) + + +def _shared_visible_declarations( + *, lookup: ScopeLookup, resource: ResourceIdentity, shared: _SharedDeclarations +) -> tuple[DeclarationRecord, ...]: + """Return non-private declarations a resource sees, reusing its directory's resolution.""" + + records: tuple[ResourceRecord, ...] = lookup.resources.get(resource, ()) + if len(records) == 1 and resource not in lookup.grants_by_resource: + directory_declarations: tuple[DeclarationRecord, ...] | None = shared.get( + _directory_key(record=records[0]) + ) + if directory_declarations is not None: + return directory_declarations + return _resolve_shared_declarations(lookup=lookup, resource=resource) + + +def _resolve_shared_declarations( + *, lookup: ScopeLookup, resource: ResourceIdentity +) -> tuple[DeclarationRecord, ...]: + resolution: VisibilityResolution = resolve_scope_visibility(lookup=lookup, target=resource) + declarations: list[DeclarationRecord] = [] + for visible in resolution.visible: + records: tuple[DeclarationRecord, ...] = lookup.declarations.get(visible.declaration, ()) + if records and records[0].scope is not ScopeKind.PRIVATE: + declarations.append(records[0]) + return tuple(declarations) + + def _tested_macro_grants( *, lookup: ScopeLookup, resource: ResourceIdentity, sql: str, file_label: str ) -> list[GrantRecord]: diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index 3a531d37fc..f428aa948d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -1291,9 +1291,23 @@ def find_macro_call_names(sql: str) -> tuple[str, ...]: ) +def find_nested_macro_call_names(sql: str) -> tuple[str, ...]: + """Return unique macro call names, including calls inside other calls' arguments.""" + + return tuple( + name + for name in _scan_sqlbuild_call_names(sql=sql, include_nested=True) + if name not in DECLARATION_REFERENCE_NAMES + ) + + def _find_sqlbuild_call_names(sql: str) -> tuple[str, ...]: """Return executable SQLBuild call names, excluding quoted and commented text.""" + return _scan_sqlbuild_call_names(sql=sql, include_nested=False) + + +def _scan_sqlbuild_call_names(*, sql: str, include_nested: bool) -> tuple[str, ...]: if MACRO_TOKEN not in sql: return () @@ -1311,7 +1325,11 @@ def _find_sqlbuild_call_names(sql: str) -> tuple[str, ...]: opening_paren_index: int = _skip_whitespace( sql=sql, start_index=macro_start_index + 1 + len(macro_name) ) - cursor = _find_matching_paren(sql=sql, opening_paren_index=opening_paren_index) + 1 + cursor = ( + opening_paren_index + 1 + if include_nested + else _find_matching_paren(sql=sql, opening_paren_index=opening_paren_index) + 1 + ) return tuple(names) diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index 1beecdc241..59eccbe38b 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -4,6 +4,7 @@ from collections import defaultdict from collections.abc import Callable, Hashable, Iterable +from functools import lru_cache from types import MappingProxyType from sqlbuild.compiler.scopes._helpers.identities import format_identity @@ -128,6 +129,7 @@ def build_lookup(*, index: ScopeIndex) -> ScopeLookup: ) +@lru_cache(maxsize=65_536) def identity_key(identity: ResourceIdentity | DeclarationIdentity) -> str: return format_identity(identity=identity) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py index f37cc51815..eacc9fa534 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py +++ b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from typing import cast -from sqlbuild.compiler.scopes._helpers.identities import format_identity +from sqlbuild.compiler.scopes._helpers.lookup import identity_key from sqlbuild.compiler.scopes._helpers.paths import normalize_path from sqlbuild.compiler.scopes.constants import ( DECLARATION_GROUP_DIRECTORY, @@ -38,7 +38,7 @@ def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: declarations: list[DeclarationRecord] = sorted( index.declarations, key=lambda item: ( - format_identity(identity=item.identity), + identity_key(item.identity), item.path, item.line, item.column, @@ -46,41 +46,41 @@ def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: ) resources: list[ResourceRecord] = sorted( index.resources, - key=lambda item: (format_identity(identity=item.identity), item.path), + key=lambda item: (identity_key(item.identity), item.path), ) grants: list[GrantRecord] = sorted( index.grants, key=lambda item: ( - format_identity(identity=item.resource), - format_identity(identity=item.declaration), - format_identity(identity=item.through), + identity_key(item.resource), + identity_key(item.declaration), + identity_key(item.through), item.kind.value, ), ) usages: list[UsageRecord] = sorted( index.usages, key=lambda item: ( - format_identity(identity=item.consumer), - format_identity(identity=item.declaration), + identity_key(item.consumer), + identity_key(item.declaration), item.kind.value, - format_identity(identity=item.through) if item.through is not None else "", + identity_key(item.through) if item.through is not None else "", item.enum_member or "", ), ) visibility: list[VisibilityRecord] = sorted( index.visibility, key=lambda item: ( - format_identity(identity=item.resource), - format_identity(identity=item.declaration), + identity_key(item.resource), + identity_key(item.declaration), item.reason.value, - format_identity(identity=item.through) if item.through is not None else "", + identity_key(item.through) if item.through is not None else "", ), ) inaccessible: list[InaccessibleRecord] = sorted( index.inaccessible, key=lambda item: ( - format_identity(identity=item.resource), - format_identity(identity=item.declaration), + identity_key(item.resource), + identity_key(item.declaration), item.reason.value, ), ) @@ -103,7 +103,7 @@ def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: ], "resources": [ { - "identity": format_identity(identity=record.identity), + "identity": identity_key(record.identity), "kind": record.identity.kind.value, "name": record.identity.name, "path": safe_scope_path(path=record.path), @@ -119,40 +119,36 @@ def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: "declarations": [_declaration_projection(record=record) for record in declarations], "grants": [ { - "resource": format_identity(identity=record.resource), - "declaration": format_identity(identity=record.declaration), - "through": format_identity(identity=record.through), + "resource": identity_key(record.resource), + "declaration": identity_key(record.declaration), + "through": identity_key(record.through), "kind": record.kind.value, } for record in grants ], "usages": [ { - "consumer": format_identity(identity=record.consumer), - "declaration": format_identity(identity=record.declaration), + "consumer": identity_key(record.consumer), + "declaration": identity_key(record.declaration), "kind": record.kind.value, - "through": ( - format_identity(identity=record.through) if record.through is not None else None - ), + "through": (identity_key(record.through) if record.through is not None else None), "enum_member": record.enum_member, } for record in usages ], "visibility": [ { - "resource": format_identity(identity=record.resource), - "declaration": format_identity(identity=record.declaration), + "resource": identity_key(record.resource), + "declaration": identity_key(record.declaration), "reason": record.reason.value, - "through": ( - format_identity(identity=record.through) if record.through is not None else None - ), + "through": (identity_key(record.through) if record.through is not None else None), } for record in visibility ], "inaccessible": [ { - "resource": format_identity(identity=record.resource), - "declaration": format_identity(identity=record.declaration), + "resource": identity_key(record.resource), + "declaration": identity_key(record.declaration), "reason": record.reason.value, } for record in inaccessible @@ -172,14 +168,12 @@ def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: "line": diagnostic.line, "column": diagnostic.column, "declaration": ( - format_identity(identity=diagnostic.declaration) + identity_key(diagnostic.declaration) if diagnostic.declaration is not None else None ), "resource": ( - format_identity(identity=diagnostic.resource) - if diagnostic.resource is not None - else None + identity_key(diagnostic.resource) if diagnostic.resource is not None else None ), } for diagnostic in sorted( @@ -201,7 +195,7 @@ def _declaration_projection(*, record: DeclarationRecord) -> dict[str, JsonValue metadata["macro"] = { "parameters": list(record.macro.parameters), "dependencies": sorted( - format_identity(identity=identity) for identity in record.macro.dependencies + identity_key(identity) for identity in record.macro.dependencies ), "source_digest": record.macro.source_digest, } @@ -220,13 +214,11 @@ def _declaration_projection(*, record: DeclarationRecord) -> dict[str, JsonValue } role, visibility, role_root, bucket_path = _declaration_container(record=record) return { - "identity": format_identity(identity=record.identity), + "identity": identity_key(record.identity), "kind": record.identity.kind.value, "name": record.identity.name, "owner": ( - format_identity(identity=record.identity.owner) - if record.identity.owner is not None - else None + identity_key(record.identity.owner) if record.identity.owner is not None else None ), "path": safe_scope_path(path=record.path), "line": record.line, @@ -275,15 +267,11 @@ def declaration_report( """Project a declaration without source, values, callables, or process data.""" usages: tuple[UsageRecord, ...] = lookup.usages_by_declaration.get(record.identity, ()) - consumers: tuple[str, ...] = tuple( - sorted({format_identity(identity=usage.consumer) for usage in usages}) - ) + consumers: tuple[str, ...] = tuple(sorted({identity_key(usage.consumer) for usage in usages})) dependencies: tuple[str, ...] = () metadata: list[tuple[str, object]] = [] if record.macro is not None: - dependencies = tuple( - sorted(format_identity(identity=item) for item in record.macro.dependencies) - ) + dependencies = tuple(sorted(identity_key(item) for item in record.macro.dependencies)) metadata.extend( (("parameters", record.macro.parameters), ("dependency_count", len(dependencies))) ) @@ -326,7 +314,7 @@ def declaration_report( grants: tuple[str, ...] = tuple( sorted( { - format_identity(identity=grant.resource) + identity_key(grant.resource) for grant in lookup.index.grants if grant.declaration == record.identity } @@ -334,14 +322,10 @@ def declaration_report( ) required_scope, required_path, promotion = required_placement(lookup=lookup, record=record) return DeclarationReport( - identity=format_identity(identity=record.identity), + identity=identity_key(record.identity), kind=record.identity.kind.value, name=record.identity.name, - owner=( - format_identity(identity=record.identity.owner) - if record.identity.owner is not None - else None - ), + owner=(identity_key(record.identity.owner) if record.identity.owner is not None else None), definition=SourceLocation(safe_scope_path(path=record.path), record.line, record.column), scope=record.scope.value, owning_path=record.owning_path, @@ -406,7 +390,7 @@ def records_for_identities( sorted( records, key=lambda item: ( - format_identity(identity=item.identity), + identity_key(item.identity), item.path, item.line, item.column, @@ -430,4 +414,4 @@ def _identity_text(*, identity: DeclarationIdentity | object | None) -> str | No typed_identity: DeclarationIdentity | ResourceIdentity = cast( "DeclarationIdentity | ResourceIdentity", identity ) - return format_identity(identity=typed_identity) + return identity_key(typed_identity) diff --git a/src/sqlbuild/compiler/scopes/_helpers/visibility.py b/src/sqlbuild/compiler/scopes/_helpers/visibility.py index b652d30dbd..429336ac5c 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/visibility.py +++ b/src/sqlbuild/compiler/scopes/_helpers/visibility.py @@ -15,6 +15,7 @@ from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + GrantRecord, InaccessibleRecord, OwnershipRoot, ResourceIdentity, @@ -66,9 +67,9 @@ def resolve_visibility( inaccessible: list[InaccessibleRecord] = [] resource: ResourceRecord for resource in query.matches: - granted_declarations: set[DeclarationIdentity] = { - grant.declaration for grant in lookup.grants_by_resource.get(resource.identity, ()) - } + grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} + for grant in lookup.grants_by_resource.get(resource.identity, ()): + grants_by_declaration.setdefault(grant.declaration, []).append(grant) declaration: DeclarationRecord for declaration in lookup.index.declarations: positive: VisibilityReason | None = _visibility_reason( @@ -76,7 +77,10 @@ def resolve_visibility( ) if positive is not None: visible.append(VisibilityRecord(resource.identity, declaration.identity, positive)) - if declaration.identity in granted_declarations: + declaration_grants: list[GrantRecord] | None = grants_by_declaration.get( + declaration.identity + ) + if declaration_grants is not None: visible.extend( VisibilityRecord( resource.identity, @@ -88,8 +92,7 @@ def resolve_visibility( ), grant.through, ) - for grant in lookup.grants_by_resource.get(resource.identity, ()) - if grant.declaration == declaration.identity + for grant in declaration_grants ) elif positive is None: inaccessible.append( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f84c13deae..3c09a4ca08 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -144,6 +144,9 @@ class FreshProcessCompileCachePerformanceGuardTestCase: expected_leaf_edit_fingerprint: str expected_macro_edit_fingerprint: str expected_project_config_fingerprint: str + macro_call_interval: int + scoped_macros: bool + expected_macro_edit_misses: int @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f726b3d421..d944ef3efa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4,6 +4,7 @@ import json import os +import re import signal import statistics import subprocess @@ -442,6 +443,8 @@ def run_fresh_process_compile_cache_benchmark( expected_cold_max_wall_seconds: float, expected_warm_max_wall_seconds: float, expected_edit_max_wall_seconds: float, + macro_call_interval: int, + scoped_macros: bool, ) -> FreshProcessCompileCacheBenchmarkResult: """Measure exact and incremental cache reuse across independent CLI processes.""" @@ -454,6 +457,8 @@ def run_fresh_process_compile_cache_benchmark( macro_count=macro_count, test_count=test_count, audit_count=audit_count, + macro_call_interval=macro_call_interval, + scoped_macros=scoped_macros, ) target_dir: Path = project_dir / "target" assert not target_dir.exists() @@ -482,7 +487,7 @@ def run_fresh_process_compile_cache_benchmark( expected_max_wall_seconds=expected_warm_max_wall_seconds, compile_args=(), ) - macro_path: Path = project_dir / "macros" / "macro_00000.py" + macro_path: Path = next(project_dir.rglob("macro_00000.py")) _replace_benchmark_text( path=macro_path, old='return f"({expression} + 1)"', @@ -1011,6 +1016,7 @@ def _test_heavy_sql(*, base_index: int, target_index: int, fixture_row_count: in _TOP_LEVEL_WITH_INTERVAL: int = 20 _NESTED_QUERY_INTERVAL: int = 15 _MACRO_INTERVAL: int = 13 +_BENCHMARK_MACRO_CALL: re.Pattern[str] = re.compile(r"@(macro_\d{5})\(") _FUNCTION_INTERVAL: int = 43 _SEED_INTERVAL: int = 67 _SEED_REFERENCE_START_INDEX: int = 300 @@ -1086,6 +1092,8 @@ def write_semantic_compile_project( macro_count: int, test_count: int, audit_count: int, + macro_call_interval: int = _MACRO_INTERVAL, + scoped_macros: bool = False, ) -> None: """Write a neutral project with broad resources and dense SQL semantics.""" @@ -1103,6 +1111,10 @@ def write_semantic_compile_project( function_count=function_count, macro_count=macro_count, audit_count=audit_count, + macro_call_interval=macro_call_interval, + ) + {True: _scope_single_folder_macros, False: _keep_project_macros}[scoped_macros]( + project_dir=project_dir ) _layered_write_tests( project_dir=project_dir, @@ -1223,6 +1235,29 @@ def macro_{index:05d}(expression: str) -> str: ) +def _keep_project_macros(*, project_dir: Path) -> None: + """Leave generated macros in the project-wide macro directory.""" + + +def _scope_single_folder_macros(*, project_dir: Path) -> None: + """Move each macro beside its only consuming folder and drop unused macros.""" + + consumer_folders: dict[str, set[str]] = {} + for model_path in sorted((project_dir / "models").rglob("*.sql")): + for call in _BENCHMARK_MACRO_CALL.findall(model_path.read_text(encoding="utf-8")): + consumer_folders.setdefault(call, set()).add(model_path.parent.name) + macros_dir: Path = project_dir / "macros" + generated: set[str] = {path.stem for path in macros_dir.glob("macro_*.py")} + for unused_stem in sorted(generated - set(consumer_folders)): + (macros_dir / f"{unused_stem}.py").unlink() + assert all(len(folders) == 1 for folders in consumer_folders.values()) + for stem, folders in sorted(consumer_folders.items()): + (folder,) = folders + scoped_dir: Path = project_dir / "models" / folder / "_sqlbuild" / "_macros" + scoped_dir.mkdir(parents=True, exist_ok=True) + (macros_dir / f"{stem}.py").rename(scoped_dir / f"{stem}.py") + + def _layered_write_schemas(*, project_dir: Path) -> None: schemas_dir: Path = project_dir / "schemas" schemas_dir.mkdir() @@ -1447,6 +1482,7 @@ def _semantic_write_models( function_count: int, macro_count: int, audit_count: int, + macro_call_interval: int = _MACRO_INTERVAL, ) -> None: for index in range(model_count): model_dir: Path = project_dir / "models" / _layered_model_folder(index=index) @@ -1469,6 +1505,7 @@ def _semantic_write_models( macro_count=macro_count, audit_count=audit_count, column_count=column_count, + macro_call_interval=macro_call_interval, ), } sql: str = builders[index == _SEMANTIC_SET_OPERATION_MODEL_INDEX]() @@ -1522,6 +1559,7 @@ def _semantic_regular_model_sql( macro_count: int, audit_count: int, column_count: int, + macro_call_interval: int = _MACRO_INTERVAL, ) -> str: header: str = _semantic_model_header( index=index, @@ -1551,11 +1589,11 @@ def _semantic_regular_model_sql( True: f'__udf("fn_{function_index:05d}")(amount)', False: "amount + CAST(@@benchmark_revision AS INTEGER)", }[index % _FUNCTION_INTERVAL == 0] - macro_index: int = (index // _MACRO_INTERVAL) % macro_count + macro_index: int = (index // macro_call_interval) % macro_count id_expression: str = { True: f'@macro_{macro_index:05d}("id")', False: "id", - }[index % _MACRO_INTERVAL == 0] + }[index % macro_call_interval == 0] direct_sql: str = f"""SELECT CAST({id_expression} AS INTEGER) AS id, CAST({amount_expression} AS DOUBLE) AS amount, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index a3a182d65e..3d18a61e88 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -39,27 +39,30 @@ source_count=713, seed_count=141, function_count=71, - macro_count=37, + macro_count=500, test_count=2_945, audit_count=5_056, expected_cold_max_wall_seconds=14.0, expected_warm_max_wall_seconds=8.0, - expected_edit_max_wall_seconds=7.0, + expected_edit_max_wall_seconds=7.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( - "6841a94fc2d3a740cb0db14fe6cee46bfe32171b9580949134cd40379a31bf94" + "36e2274af2bd6840f41d88ee1140a91034505a50f7fa27b61e2e47aa06765314" ), expected_leaf_edit_fingerprint=( - "a58095a0cd5c2866cd99df92c2503174b400a136379b68dc5889272cef30f17e" + "f6cfcbb21bf7e011b76f08f5d8e61b2095316b6845d987090e79b1718ed54175" ), expected_macro_edit_fingerprint=( - "fdef695e0e1d08cb3809bb0589b0c535b1bbb604bf81a428627dce09317911f3" + "ae102976a43c0fd6a68e5e55a7feeb9fac5f49ffc56bd3cdfcd389e199582081" ), expected_project_config_fingerprint=( - "990efe5eee76ba8e0690e5203264aa948e59c39e12463120a3661ea976f9ec8d" + "96e996a556119f452e966096747b35219941bf7bd324624242aadf6375ba1a31" ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, ), FreshProcessCompileCachePerformanceGuardTestCase( description="models_5000_fresh_process_compile_cache_stays_incremental", @@ -67,27 +70,30 @@ source_count=1_189, seed_count=236, function_count=118, - macro_count=61, + macro_count=834, test_count=4_908, audit_count=8_427, expected_cold_max_wall_seconds=23.0, - expected_warm_max_wall_seconds=12.0, - expected_edit_max_wall_seconds=11.0, + expected_warm_max_wall_seconds=14.0, + expected_edit_max_wall_seconds=13.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( - "32dac15ee9c15a2aea2e4783e888231e402f15a8fe83270ef55af1aa75924b6c" + "686664bcadbbb0e339205d71b8fb3ab2d3f1226ca888eb4b2e6a5aa97ade1efb" ), expected_leaf_edit_fingerprint=( - "4b1d9da0a0141ded2db0cfb7719fb0ccc12353f129638822e451c11e98f55724" + "43fa1db2d5db12c5031aef8f6fb2a0e4af89c51ece58b1f1772db0a9ea4d43a0" ), expected_macro_edit_fingerprint=( - "3a8d7e4ebe24a354f4d9f470e4b37b0286cf12a2f36f850053651c17d9f8e3b3" + "fd0cc80586b11d8d79a904d1daa6da4d559d22d6b4ebea02139a7d12626b770f" ), expected_project_config_fingerprint=( - "c1efca198bc601d5135898ad059cfb6e016872e15f216cc357662bb185ba29f3" + "44a292e9ada77cc7038036fb593d9d8d8cd4883062a07254d963963ed0b4bb45" ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, ), FreshProcessCompileCachePerformanceGuardTestCase( description="models_10000_fresh_process_compile_cache_stays_incremental", @@ -95,27 +101,30 @@ source_count=2_377, seed_count=471, function_count=236, - macro_count=123, + macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=42.5, - expected_warm_max_wall_seconds=24.0, - expected_edit_max_wall_seconds=20.5, + expected_cold_max_wall_seconds=46.5, + expected_warm_max_wall_seconds=27.0, + expected_edit_max_wall_seconds=24.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( - "7e1410c35f6026c7520711224aea2a95d8fe46515a2d2d370a3ee21180a36fdc" + "c1364cae404ee768c4443c3b964c42370e33268fbf83b101890b13558290e3e0" ), expected_leaf_edit_fingerprint=( - "059dc35c437adacfe0098e0cc19ccfeaae9fc5ca9aa682721cd3f37cbff92f77" + "fe35feb1b239a17ed0e7b22bca20c2af06e0f25335fe5f9fae212491a693911b" ), expected_macro_edit_fingerprint=( - "e2cfd1b086c42ce4d181380a3be1d7a3d1360ee7ccea6f5dfbb5a8d2af5a76ce" + "bec17e83316ca926b76f450fa8a035e83f19c061e14a11b8effe04b7fa2e26fd" ), expected_project_config_fingerprint=( - "38323964889d18747f10614c8442d9a2cdd9d38efed88369cb10c3bf1099ed85" + "780e26aa973459fbc803dd8d64315d445aab8bc57e90b0ad1ecb94f074b22474" ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, ), ], ids=lambda case: case.description, @@ -137,6 +146,8 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in expected_cold_max_wall_seconds=test_case.expected_cold_max_wall_seconds, expected_warm_max_wall_seconds=test_case.expected_warm_max_wall_seconds, expected_edit_max_wall_seconds=test_case.expected_edit_max_wall_seconds, + macro_call_interval=test_case.macro_call_interval, + scoped_macros=test_case.scoped_macros, ) measurements: dict[str, FreshProcessCompileBenchmarkResult] = { "cold": result.cold, @@ -214,7 +225,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in assert macro_batch_hits + macro_entry_hits + macro_misses == test_case.model_count assert macro_batch_hits > 0 assert macro_entry_hits > 0 - assert macro_misses == 7 + assert macro_misses == test_case.expected_macro_edit_misses assert macro_bypasses == 0 assert_complete_compile_cache_hit( measurement=result.after_macro_edit, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index efec5d2d4b..1295b17eab 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -435,6 +435,49 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ expected_sql_fragments=("SELECT 6 + 7 AS value",), expected_grants=(("constant:model_value", "model:orders", "test:check"),), ), + ExpectedModelDeclarationGrantTestCase( + description="only macros called by the test are granted", + files={ + "models/domain/_sqlbuild/_macros/discount.py": ( + 'def discount(expression: str) -> str:\n return f"({expression} - 1)"\n' + ), + "models/domain/_sqlbuild/_macros/surcharge.py": ( + 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' + ), + "models/domain/orders.sql": ( + "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + ), + "tests/unit/check.sql": ( + "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "__expected__orders AS (SELECT @discount('10') AS value) SELECT 1" + ), + }, + expected_sql_fragments=("SELECT (10 - 1) AS value",), + expected_grants=(("macro:discount", "model:orders", "test:check"),), + ), + ExpectedModelDeclarationGrantTestCase( + description="macros called inside macro arguments are granted", + files={ + "models/domain/_sqlbuild/_macros/discount.py": ( + 'def discount(expression: str) -> str:\n return f"({expression} - 1)"\n' + ), + "models/domain/_sqlbuild/_macros/surcharge.py": ( + 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' + ), + "models/domain/orders.sql": ( + "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + ), + "tests/unit/check.sql": ( + "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "__expected__orders AS (SELECT @discount(@surcharge('10')) AS value) SELECT 1" + ), + }, + expected_sql_fragments=("SELECT ((10 + 1) - 1) AS value",), + expected_grants=( + ("macro:discount", "model:orders", "test:check"), + ("macro:surcharge", "model:orders", "test:check"), + ), + ), ExpectedModelDeclarationGrantTestCase( description="model private declaration is not granted", files={