From 4f3fe8467b4ab351752b797226a556937ea39692 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:53:53 +0100 Subject: [PATCH 1/4] perf(compile): index scope visibility by declaration scope Group declarations by scope and owner once per scope lookup so a resource's visible declarations come from owner and ancestor lookups instead of classifying every declaration. Compile callers only need the identities of inaccessible declarations, so they use a resolver that skips per-pair inaccessible reasons. The full resolver keeps its output. --- .../attachment/scope_relationships.py | 12 +- .../compile/_helpers/render/declarations.py | 10 +- .../compiler/scopes/_helpers/lookup.py | 2 + .../compiler/scopes/_helpers/visibility.py | 214 +++++++++++--- .../_resolve_scope_declaration_visibility.py | 16 ++ src/sqlbuild/compiler/scopes/models.py | 21 ++ .../sqlbuild/compiler/scopes/_test_helpers.py | 260 ++++++++++++++++++ .../sqlbuild/compiler/scopes/_test_types.py | 12 + .../compiler/scopes/test_visibility_index.py | 74 +++++ 9 files changed, 576 insertions(+), 45 deletions(-) create mode 100644 src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py create mode 100644 tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 1593bb20b8..b1f921b905 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -29,20 +29,22 @@ from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( resolve_scope_path_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, GrantRecord, ResourceIdentity, ResourceRecord, ScopeIndex, ScopeLookup, - VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -288,7 +290,9 @@ def _shared_visible_declarations( def _resolve_shared_declarations( *, lookup: ScopeLookup, resource: ResourceIdentity ) -> tuple[DeclarationRecord, ...]: - resolution: VisibilityResolution = resolve_scope_visibility(lookup=lookup, target=resource) + resolution: DeclarationVisibility = resolve_scope_declaration_visibility( + lookup=lookup, target=resource + ) declarations: list[DeclarationRecord] = [] for visible in resolution.visible: records: tuple[DeclarationRecord, ...] = lookup.declarations.get(visible.declaration, ()) @@ -314,7 +318,7 @@ def _tested_macro_grants( if actual_cte is None: return [] tested_macro_names: tuple[str, ...] = find_macro_call_names(actual_cte.sql_body) - direct_resolution: VisibilityResolution = resolve_scope_visibility( + direct_resolution: DeclarationVisibility = resolve_scope_declaration_visibility( lookup=lookup, target=resource ) directly_visible: frozenset[DeclarationIdentity] = frozenset( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index e6c27ec574..c9678e7cb9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -34,20 +34,22 @@ from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( resolve_scope_path_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, ResourceIdentity, ResourceRecord, ScopeIndex, UsageRecord, VisibilityRecord, - VisibilityResolution, ) from sqlbuild.compiler.scopes.types import ( DeclarationKind, @@ -197,7 +199,7 @@ def resolve_declaration_context( ) if cached_path_context is not None: return cached_path_context - resolution: VisibilityResolution = resolve_scope_visibility( + resolution: DeclarationVisibility = resolve_scope_declaration_visibility( lookup=resolver.lookup, target=resource or target_path ) enums: dict[str, EnumDeclaration] = {} @@ -218,7 +220,7 @@ def resolve_declaration_context( resolver.lookup.declarations[item.declaration][0] for item in resolution.visible ) inaccessible_records: tuple[DeclarationRecord, ...] = tuple( - resolver.lookup.declarations[item.declaration][0] for item in resolution.inaccessible + resolver.lookup.declarations[identity][0] for identity in resolution.inaccessible ) for visible_record in resolution.visible: visibility_by_declaration.setdefault(visible_record.declaration, []).append( diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index f9d2453e32..4265621753 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.scopes._helpers.identities import format_identity from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes._helpers.visibility import build_visibility_index from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -116,6 +117,7 @@ def build_lookup(*, index: ScopeIndex) -> ScopeLookup: key=lambda item: item.resource, sort_key=grant_sort_key, ), + visibility_index=build_visibility_index(declarations=canonical_index.declarations), ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/visibility.py b/src/sqlbuild/compiler/scopes/_helpers/visibility.py index 6bed2d1e14..facd691eb2 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/visibility.py +++ b/src/sqlbuild/compiler/scopes/_helpers/visibility.py @@ -4,6 +4,7 @@ from functools import lru_cache from pathlib import PurePath +from types import MappingProxyType from sqlbuild.compiler.scopes._helpers.identities import parse_identity from sqlbuild.compiler.scopes._helpers.paths import normalize_path @@ -15,6 +16,8 @@ from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, + DeclarationVisibilityIndex, GrantRecord, InaccessibleRecord, OwnershipRoot, @@ -62,49 +65,186 @@ def query_target( def resolve_visibility( *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> VisibilityResolution: + query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) + declarations: tuple[DeclarationRecord, ...] = lookup.index.declarations + inaccessible: list[InaccessibleRecord] = [] + resource: ResourceRecord + positions: list[int] + for resource, positions in inaccessible_groups: + inaccessible.extend( + InaccessibleRecord( + resource.identity, + declarations[position].identity, + _inaccessible_reason(resource=resource, declaration=declarations[position]), + ) + for position in positions + ) + return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + + +def resolve_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + """Resolve visible facts and inaccessible identities without per-pair inaccessible reasons.""" + + query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) + identities: tuple[DeclarationIdentity, ...] = lookup.visibility_index.identities + inaccessible: list[DeclarationIdentity] = [] + positions: list[int] + for _resource, positions in inaccessible_groups: + inaccessible.extend(identities[position] for position in positions) + return DeclarationVisibility(query, tuple(visible), tuple(inaccessible)) + + +def _classify_target( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> tuple[ScopeTargetQuery, list[VisibilityRecord], list[tuple[ResourceRecord, list[int]]]]: + """Return the query, ordered visible records, and inaccessible positions per match.""" + query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - if query.unknown: - return VisibilityResolution(target=query) visible: list[VisibilityRecord] = [] - inaccessible: list[InaccessibleRecord] = [] + inaccessible_groups: list[tuple[ResourceRecord, list[int]]] = [] resource: ResourceRecord for resource in query.matches: - grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} - for grant in lookup.grants_by_resource.get(resource.identity, ()): - grants_by_declaration.setdefault(grant.declaration, []).append(grant) - declaration: DeclarationRecord - for declaration in lookup.index.declarations: - positive: VisibilityReason | None = _visibility_reason( - resource=resource, declaration=declaration + resource_visible, positions = _classify_resource(lookup=lookup, resource=resource) + visible.extend(resource_visible) + inaccessible_groups.append((resource, positions)) + return query, visible, inaccessible_groups + + +def _classify_resource( + *, lookup: ScopeLookup, resource: ResourceRecord +) -> tuple[list[VisibilityRecord], list[int]]: + """Return ordered visible records and the declaration positions left inaccessible.""" + + index: DeclarationVisibilityIndex = lookup.visibility_index + positive: list[tuple[int, VisibilityReason]] = _visible_positions( + index=index, resource=resource + ) + grants: tuple[GrantRecord, ...] = lookup.grants_by_resource.get(resource.identity, ()) + if not grants: + visible_positions: set[int] = {position for position, _reason in positive} + return ( + [ + VisibilityRecord(resource.identity, index.identities[position], reason) + for position, reason in positive + ], + [ + position + for position in range(len(index.identities)) + if position not in visible_positions + ], + ) + reasons: dict[int, VisibilityReason] = dict(positive) + grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} + for grant in grants: + grants_by_declaration.setdefault(grant.declaration, []).append(grant) + visible: list[VisibilityRecord] = [] + inaccessible: list[int] = [] + position: int + identity: DeclarationIdentity + for position, identity in enumerate(index.identities): + reason: VisibilityReason | None = reasons.get(position) + if reason is not None: + visible.append(VisibilityRecord(resource.identity, identity, reason)) + declaration_grants: list[GrantRecord] | None = grants_by_declaration.get(identity) + if declaration_grants is not None: + visible.extend( + VisibilityRecord( + resource.identity, + identity, + ( + VisibilityReason.TESTED_MACRO + if grant.kind is GrantKind.TESTED_MACRO + else VisibilityReason.EXPECTED_MODEL + ), + grant.through, + ) + for grant in declaration_grants ) - if positive is not None: - visible.append(VisibilityRecord(resource.identity, declaration.identity, positive)) - declaration_grants: list[GrantRecord] | None = grants_by_declaration.get( - declaration.identity + elif reason is None: + inaccessible.append(position) + return visible, inaccessible + + +def build_visibility_index( + *, declarations: tuple[DeclarationRecord, ...] +) -> DeclarationVisibilityIndex: + """Group canonical declaration positions by the scope facts that make them visible.""" + + global_positions: list[int] = [] + private: dict[ResourceIdentity, list[int]] = {} + local: dict[str, list[int]] = {} + inherited: dict[str, list[int]] = {} + position: int + declaration: DeclarationRecord + for position, declaration in enumerate(declarations): + if declaration.scope is ScopeKind.GLOBAL: + global_positions.append(position) + elif declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None: + private.setdefault(declaration.identity.owner, []).append(position) + elif declaration.scope is ScopeKind.LOCAL: + local.setdefault(_declaration_owner(declaration), []).append(position) + elif declaration.scope is ScopeKind.INHERITED: + inherited.setdefault(_declaration_owner(declaration), []).append(position) + return DeclarationVisibilityIndex( + identities=tuple(declaration.identity for declaration in declarations), + global_positions=tuple(global_positions), + private_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in private.items()} + ), + local_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in local.items()} + ), + inherited_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in inherited.items()} + ), + ) + + +def _visible_positions( + *, index: DeclarationVisibilityIndex, resource: ResourceRecord +) -> list[tuple[int, VisibilityReason]]: + """Return declaration positions visible to a resource, in declaration order.""" + + positive: list[tuple[int, VisibilityReason]] = [ + (position, VisibilityReason.GLOBAL) for position in index.global_positions + ] + positive.extend( + (position, VisibilityReason.PRIVATE_OWNER) + for position in index.private_positions.get(resource.identity, ()) + ) + if index.local_positions or index.inherited_positions: + parent: str = _parent(resource.path) + positive.extend( + (position, VisibilityReason.LOCAL_OWNER) + for position in index.local_positions.get(parent, ()) + ) + ancestor: str + for ancestor in _canonical_ancestors(parent): + positive.extend( + (position, VisibilityReason.INHERITED_ANCESTOR) + for position in index.inherited_positions.get(ancestor, ()) ) - if declaration_grants is not None: - visible.extend( - VisibilityRecord( - resource.identity, - declaration.identity, - ( - VisibilityReason.TESTED_MACRO - if grant.kind is GrantKind.TESTED_MACRO - else VisibilityReason.EXPECTED_MODEL - ), - grant.through, - ) - for grant in declaration_grants - ) - elif positive is None: - inaccessible.append( - InaccessibleRecord( - resource.identity, - declaration.identity, - _inaccessible_reason(resource=resource, declaration=declaration), - ) - ) - return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + positive.sort() + return positive + + +def _declaration_owner(declaration: DeclarationRecord) -> str: + return _canonical_owner(declaration.owning_path or CURRENT_PATH_COMPONENT) + + +@lru_cache(maxsize=65_536) +def _canonical_ancestors(path: str) -> tuple[str, ...]: + """Return every owner that ``_is_canonical_descendant`` accepts as an ancestor of ``path``.""" + + prefixes: list[str] = [CURRENT_PATH_COMPONENT] + index: int = path.find(PATH_SEPARATOR) + while index >= 0: + prefixes.append(path[:index]) + index = path.find(PATH_SEPARATOR, index + 1) + prefixes.append(path) + return tuple(dict.fromkeys(prefixes)) def resolve_path_visibility( diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py new file mode 100644 index 0000000000..d4fe55c279 --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py @@ -0,0 +1,16 @@ +"""Resolve static declaration visibility without per-declaration inaccessible reasons.""" + +from __future__ import annotations + +from pathlib import PurePath + +from sqlbuild.compiler.scopes._helpers.visibility import resolve_declaration_visibility +from sqlbuild.compiler.scopes.models import DeclarationVisibility, ResourceIdentity, ScopeLookup + + +def resolve_scope_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + """Return visible facts and inaccessible declaration identities for one target.""" + + return resolve_declaration_visibility(lookup=lookup, target=target) diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 6f46e02a7e..97ee535dd8 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -208,6 +208,17 @@ class ScopeIndex: completeness: ScopeCompleteness = field(default_factory=ScopeCompleteness) +@dataclass(frozen=True) +class DeclarationVisibilityIndex: + """Declaration positions grouped by the scope facts that make them visible.""" + + identities: tuple[DeclarationIdentity, ...] + global_positions: tuple[int, ...] + private_positions: Mapping[ResourceIdentity, tuple[int, ...]] + local_positions: Mapping[str, tuple[int, ...]] + inherited_positions: Mapping[str, tuple[int, ...]] + + @dataclass(frozen=True) class ScopeLookup: """Immutable indexes over one canonical ``ScopeIndex``.""" @@ -219,6 +230,7 @@ class ScopeLookup: usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] + visibility_index: DeclarationVisibilityIndex @dataclass(frozen=True) @@ -245,6 +257,15 @@ class VisibilityResolution: inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class DeclarationVisibility: + """Visible facts and inaccessible declaration identities for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) + inaccessible: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) + + @dataclass(frozen=True) class ScopeReportFilters: """Orthogonal declaration filters applied in their documented fixed order.""" diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py new file mode 100644 index 0000000000..99712b3c65 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -0,0 +1,260 @@ +"""Reference visibility classification and randomized scope projects for equivalence tests.""" + +from __future__ import annotations + +import posixpath +import random +from collections.abc import Callable +from itertools import compress +from pathlib import PurePath + +from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes._helpers.visibility import query_target +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) +from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + DeclarationVisibility, + GrantRecord, + InaccessibleRecord, + OwnershipRoot, + ResourceIdentity, + ResourceRecord, + ScopeIndex, + ScopeLookup, + ScopeTargetQuery, + VisibilityRecord, + VisibilityResolution, +) +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + GrantKind, + InaccessibleReason, + ResourceKind, + ScopeKind, + VisibilityReason, +) + +_FOLDERS: tuple[str, ...] = ("sales", "sale", "finance", "daily", "orders") +_ROOTS: tuple[str, ...] = ("models", "tests/unit", "functions/sql") +_NAMES: tuple[str, ...] = ("status", "region", "tier", "limit", "currency", "channel") +_NOISY_SPELLINGS: tuple[Callable[[random.Random, str], str], ...] = ( + lambda _rng, path: path.replace("/", "\\"), + lambda _rng, path: f"./{path}/", + lambda rng, path: f"{path}/{rng.choice(_FOLDERS)}/..", + lambda _rng, path: path, +) +_VISIBLE_REASONS: dict[ScopeKind, VisibilityReason] = { + ScopeKind.GLOBAL: VisibilityReason.GLOBAL, + ScopeKind.PRIVATE: VisibilityReason.PRIVATE_OWNER, + ScopeKind.LOCAL: VisibilityReason.LOCAL_OWNER, + ScopeKind.INHERITED: VisibilityReason.INHERITED_ANCESTOR, +} +_GRANT_REASONS: dict[GrantKind, VisibilityReason] = { + kind: VisibilityReason.EXPECTED_MODEL for kind in GrantKind +} | {GrantKind.TESTED_MACRO: VisibilityReason.TESTED_MACRO} + + +def random_scope_lookup( + *, seed: int, resource_count: int, declaration_count: int, grant_count: int +) -> ScopeLookup: + """Build a seeded project with every scope tier, noisy owner paths, and grants.""" + + rng: random.Random = random.Random(seed) + resources: list[ResourceRecord] = [] + for index in range(resource_count): + directory: str = _directory(rng) + name: str = f"model_{index % max(1, resource_count - 3)}" + resources.append( + ResourceRecord( + ResourceIdentity(ResourceKind.MODEL, name), + f"{directory}/{name}.sql", + OwnershipRoot(directory.split("/")[0], resource_kind=ResourceKind.MODEL), + ) + ) + identities: list[ResourceIdentity] = [resource.identity for resource in resources] + owners: list[ResourceIdentity] = [*identities, ResourceIdentity(ResourceKind.MODEL, "other")] + declarations: list[DeclarationRecord] = [] + for _index in range(declaration_count): + scope: ScopeKind = rng.choice(tuple(ScopeKind)) + owner_directory: str = _directory(rng) + owning_path: str = rng.choice(_NOISY_SPELLINGS)(rng, owner_directory) + declarations.append( + DeclarationRecord( + DeclarationIdentity( + rng.choice((DeclarationKind.ENUM, DeclarationKind.MACRO)), + rng.choice(_NAMES), + {ScopeKind.PRIVATE: rng.choice(owners)}.get(scope), + ), + f"{owner_directory}/_sqlbuild/enums/{rng.randint(0, 9)}.sql", + rng.randint(1, 5), + 1, + scope, + OwnershipRoot(owner_directory.split("/")[0], resource_kind=ResourceKind.MODEL), + owning_path={True: None, False: owning_path}[ + scope is ScopeKind.GLOBAL or rng.random() < 0.1 + ], + ) + ) + grants: list[GrantRecord] = [ + GrantRecord( + rng.choice(identities), + rng.choice(declarations).identity, + rng.choice(identities), + rng.choice(tuple(GrantKind)), + ) + for _index in range(grant_count * bool(declarations and identities)) + ] + return build_scope_lookup( + index=ScopeIndex( + resources=tuple(resources), declarations=tuple(declarations), grants=tuple(grants) + ) + ) + + +def visibility_mismatches(*, lookup: ScopeLookup) -> tuple[str, ...]: + """Return every target whose indexed resolution differs from the exhaustive reference.""" + + targets: list[ResourceIdentity | str] = _targets(lookup) + differs: list[bool] = [ + resolve_scope_visibility(lookup=lookup, target=target) + != _reference_resolution(lookup=lookup, target=target) + for target in targets + ] + return tuple(map(str, compress(targets, differs))) + + +def declaration_visibility_mismatches(*, lookup: ScopeLookup) -> tuple[str, ...]: + """Return every target whose reason-free resolution differs from the exhaustive reference.""" + + targets: list[ResourceIdentity | str] = _targets(lookup) + differs: list[bool] = [ + resolve_scope_declaration_visibility(lookup=lookup, target=target) + != _reference_declaration_visibility(lookup=lookup, target=target) + for target in targets + ] + return tuple(map(str, compress(targets, differs))) + + +def _directory(rng: random.Random) -> str: + root: str = rng.choice(_ROOTS) + depth: int = rng.randint(0, 3) + return "/".join([root, *(rng.choice(_FOLDERS) for _ in range(depth))]) + + +def _targets(lookup: ScopeLookup) -> list[ResourceIdentity | str]: + return [ + *lookup.resources, + *(resource.path for resource in lookup.index.resources), + "models/missing.sql", + "model:missing", + ] + + +def _parent(path: str) -> str: + return posixpath.dirname(normalize_path(path=path)) or "." + + +def _is_descendant(*, path: str, ancestor: str) -> bool: + return ancestor == "." or path == ancestor or path.startswith(f"{ancestor}/") + + +def _reference_reason( + *, resource: ResourceRecord, declaration: DeclarationRecord +) -> VisibilityReason | None: + owner: str = normalize_path(path=declaration.owning_path or ".") + parent: str = _parent(resource.path) + visible: dict[ScopeKind, bool] = { + ScopeKind.GLOBAL: True, + ScopeKind.PRIVATE: declaration.identity.owner == resource.identity, + ScopeKind.LOCAL: parent == owner, + ScopeKind.INHERITED: _is_descendant(path=parent, ancestor=owner), + } + return {True: _VISIBLE_REASONS[declaration.scope], False: None}[visible[declaration.scope]] + + +def _reference_inaccessible_reason( + *, resource: ResourceRecord, declaration: DeclarationRecord +) -> InaccessibleReason: + owner: str = normalize_path(path=declaration.owning_path or ".") + candidates: tuple[tuple[bool, InaccessibleReason], ...] = ( + (declaration.scope is ScopeKind.PRIVATE, InaccessibleReason.PRIVATE_OWNER), + (declaration.scope is ScopeKind.LOCAL, InaccessibleReason.LOCAL_BOUNDARY), + ( + _is_descendant(path=owner, ancestor=_parent(resource.path)), + InaccessibleReason.DESCENDANT_SCOPE, + ), + ( + declaration.ownership_root.path == resource.ownership_root.path, + InaccessibleReason.SIBLING_SCOPE, + ), + (True, InaccessibleReason.UNRELATED_SCOPE), + ) + return next( + compress( + (reason for _matches, reason in candidates), + (matches for matches, _reason in candidates), + ) + ) + + +def _reference_resolution( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> VisibilityResolution: + """Classify every declaration against every matched resource, as before indexing.""" + + query: ScopeTargetQuery = query_target(lookup=lookup, target=target) + visible: list[VisibilityRecord] = [] + inaccessible: list[InaccessibleRecord] = [] + for resource in query.matches: + grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} + for grant in lookup.grants_by_resource.get(resource.identity, ()): + grants_by_declaration.setdefault(grant.declaration, []).append(grant) + for declaration in lookup.index.declarations: + positive: VisibilityReason | None = _reference_reason( + resource=resource, declaration=declaration + ) + granted: list[GrantRecord] | None = grants_by_declaration.get(declaration.identity) + visible.extend( + VisibilityRecord(resource.identity, declaration.identity, reason) + for reason in filter(None, (positive,)) + ) + visible.extend( + VisibilityRecord( + resource.identity, + declaration.identity, + _GRANT_REASONS[grant.kind], + grant.through, + ) + for grant in granted or () + ) + inaccessible.extend( + compress( + [ + InaccessibleRecord( + resource.identity, + declaration.identity, + _reference_inaccessible_reason( + resource=resource, declaration=declaration + ), + ) + ], + [granted is None and positive is None], + ) + ) + return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + + +def _reference_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + expected: VisibilityResolution = _reference_resolution(lookup=lookup, target=target) + return DeclarationVisibility( + expected.target, + expected.visible, + tuple(record.declaration for record in expected.inaccessible), + ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 785f725133..feea050209 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -229,3 +229,15 @@ class RelocationCase: consumer_paths: tuple[str, ...] destination: str expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class RandomScopeCase: + """One seeded randomized scope project compared with the exhaustive classification.""" + + description: str + seed: int + resource_count: int + declaration_count: int + grant_count: int + expected_mismatches: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py b/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py new file mode 100644 index 0000000000..c9452910f6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py @@ -0,0 +1,74 @@ +"""Equivalence of indexed scope visibility with the exhaustive per-declaration classification.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.compiler.scopes._test_helpers import ( + declaration_visibility_mismatches, + random_scope_lookup, + visibility_mismatches, +) +from tests.unit.src.sqlbuild.compiler.scopes._test_types import RandomScopeCase + + +@pytest.mark.parametrize( + "test_case", + [ + RandomScopeCase("sparse", seed=1, resource_count=12, declaration_count=20, grant_count=0), + RandomScopeCase("mixed", seed=2, resource_count=30, declaration_count=60, grant_count=6), + RandomScopeCase("dense", seed=3, resource_count=50, declaration_count=120, grant_count=15), + RandomScopeCase( + "no_declarations", seed=4, resource_count=8, declaration_count=0, grant_count=0 + ), + RandomScopeCase( + "grant_heavy", seed=5, resource_count=40, declaration_count=80, grant_count=40 + ), + RandomScopeCase("wide", seed=6, resource_count=25, declaration_count=200, grant_count=3), + ], + ids=lambda case: case.description, +) +def test_given_randomized_scopes_when_resolving_visibility_then_matches_exhaustive_classification( + test_case: RandomScopeCase, +) -> None: + mismatches: tuple[str, ...] = visibility_mismatches( + lookup=random_scope_lookup( + seed=test_case.seed, + resource_count=test_case.resource_count, + declaration_count=test_case.declaration_count, + grant_count=test_case.grant_count, + ) + ) + + assert mismatches == test_case.expected_mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + RandomScopeCase("sparse", seed=1, resource_count=12, declaration_count=20, grant_count=0), + RandomScopeCase("mixed", seed=2, resource_count=30, declaration_count=60, grant_count=6), + RandomScopeCase( + "grant_heavy", seed=5, resource_count=40, declaration_count=80, grant_count=40 + ), + RandomScopeCase("wide", seed=6, resource_count=25, declaration_count=200, grant_count=3), + ], + ids=lambda case: case.description, +) +def test_given_randomized_scopes_when_resolving_declarations_then_matches_exhaustive_identities( + test_case: RandomScopeCase, +) -> None: + mismatches: tuple[str, ...] = declaration_visibility_mismatches( + lookup=random_scope_lookup( + seed=test_case.seed, + resource_count=test_case.resource_count, + declaration_count=test_case.declaration_count, + grant_count=test_case.grant_count, + ) + ) + + assert mismatches == test_case.expected_mismatches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 57225e93081dda765823d9cfdab0cf31b2b805e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:19:35 +0100 Subject: [PATCH 2/4] perf(compile): walk discovery roots once and match model headers natively Discovery queried the project tree with dozens of recursive globs per compile: once per declaration kind, once per layout validator, and once per resource category. A discovery pass now reads each directory listing once into a scoped snapshot that answers every glob with pathlib's matching and symlink semantics, memoizes the declaration facts and group layout, and classifies scoped-tree membership once per directory. The snapshot lives only for one pass, so later passes see current files. MODEL headers are matched by an exact native equivalent of the possessive header pattern in one batch call. --- .../src/bindings/_helpers/functions.rs | 14 ++ .../_helpers/model_headers/matching.rs | 145 ++++++++++++ .../compiler/_helpers/model_headers/mod.rs | 1 + .../src/compiler/main/mod.rs | 1 + .../compiler/main/model_header_matching.rs | 8 + .../src/compiler/tests.rs | 2 + .../tests/test_model_header_matching.rs | 72 ++++++ .../src/compiler/tests/test_types.rs | 6 + src/sqlbuild/_native.pyi | 1 + .../_helpers/filesystem/aggregation.py | 15 +- .../discovery/_helpers/filesystem/core.py | 211 ++++++++--------- .../_helpers/filesystem/model_files.py | 5 +- .../_helpers/filesystem/named_declarations.py | 27 ++- .../_helpers/filesystem/scoped_paths.py | 68 ++++++ .../discovery/_helpers/sql/model_files.py | 59 +++-- .../discovery/classes/directory_snapshot.py | 135 +++++++++++ src/sqlbuild/compiler/discovery/models.py | 22 ++ src/sqlbuild/compiler/discovery/types.py | 10 +- src/sqlbuild/compiler/scopes/constants.py | 3 + .../discovery/_helpers/_test_helpers.py | 217 ++++++++++++++++++ .../discovery/_helpers/_test_types.py | 39 ++++ .../test_filesystem_directory_snapshot.py | 147 ++++++++++++ 22 files changed, 1071 insertions(+), 137 deletions(-) create mode 100644 crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs create mode 100644 crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs create mode 100644 crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py create mode 100644 src/sqlbuild/compiler/discovery/classes/directory_snapshot.py create mode 100644 tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 59b33f5732..260f595971 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -376,6 +376,19 @@ fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult, + contents: Vec, +) -> PyResult>> { + py.compiler_detach(|| { + Ok(crate::compiler::main::model_header_matching::match_batch( + &contents, + )) + }) + .map_err(value_error) +} + #[pyfunction] fn tokenize_model_header(header: &str) -> PyResult> { compiler_guard(|| { @@ -492,6 +505,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; + module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?; module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs new file mode 100644 index 0000000000..df18220cc1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs @@ -0,0 +1,145 @@ +//! Exact native equivalent of the possessive Python `MODEL(...)` header regular expression. + +/// Code-point offsets of one matched header: header start, header end, and SQL start. +pub(crate) type HeaderMatchOffsets = (usize, usize, usize); + +const MODEL_KEYWORD: &[u8] = b"MODEL"; + +pub(crate) fn match_one(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut index = skip_whitespace(text, 0); + if !bytes[index..].starts_with(MODEL_KEYWORD) { + return None; + } + index = skip_whitespace(text, index + MODEL_KEYWORD.len()); + if bytes.get(index) != Some(&b'(') { + return None; + } + let header_start = index + 1; + let header_end = scan_header_body(text, header_start); + if bytes.get(header_end) != Some(&b')') { + return None; + } + index = skip_whitespace(text, header_end + 1); + if bytes.get(index) != Some(&b';') { + return None; + } + let sql_start = skip_whitespace(text, index + 1); + Some(code_point_offsets( + text, + header_start, + header_end, + sql_start, + )) +} + +/// Consume the possessive header body and return the byte offset where it stops. +fn scan_header_body(text: &str, start: usize) -> usize { + let bytes = text.as_bytes(); + let mut index = start; + while let Some(&byte) = bytes.get(index) { + index = match byte { + b'"' | b'\'' => quoted_string_end(text, index, byte) + .unwrap_or_else(|| unclosed_quote_end(text, index)), + b')' => { + let after = skip_whitespace(text, index + 1); + if bytes.get(after) == Some(&b';') { + return index; + } + index + 1 + } + b'\\' => match escape_end(text, index) { + Some(end) => end, + None => return index, + }, + _ => index + 1, + }; + } + index +} + +/// `"(?:[^"\\]|\\.)*+"`: the closing quote's end, or `None` when the string never closes. +fn quoted_string_end(text: &str, open: usize, quote: u8) -> Option { + let bytes = text.as_bytes(); + let mut index = open + 1; + while let Some(&byte) = bytes.get(index) { + if byte == quote { + return Some(index + 1); + } + if byte == b'\\' { + index = escape_end(text, index)?; + } else { + index += 1; + } + } + None +} + +/// `["'](?:[^\\)]|\\.)*+`: an unclosed quote runs to the next `)` or unescapable backslash. +fn unclosed_quote_end(text: &str, open: usize) -> usize { + let bytes = text.as_bytes(); + let mut index = open + 1; + while let Some(&byte) = bytes.get(index) { + match byte { + b')' => return index, + b'\\' => match escape_end(text, index) { + Some(end) => index = end, + None => return index, + }, + _ => index += 1, + } + } + index +} + +/// `\\.` with DOTALL: a backslash plus one following code point, if any. +fn escape_end(text: &str, backslash: usize) -> Option { + text[backslash + 1..] + .chars() + .next() + .map(|escaped| backslash + 1 + escaped.len_utf8()) +} + +fn skip_whitespace(text: &str, start: usize) -> usize { + let mut index = start; + for character in text[start..].chars() { + if !is_python_whitespace(character) { + break; + } + index += character.len_utf8(); + } + index +} + +/// Python's `str.isspace()`, which `\s` uses for `str` patterns. +fn is_python_whitespace(character: char) -> bool { + matches!( + character, + '\t'..='\r' + | '\u{1c}'..='\u{20}' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} + +fn code_point_offsets( + text: &str, + header_start: usize, + header_end: usize, + sql_start: usize, +) -> HeaderMatchOffsets { + if text.is_ascii() { + return (header_start, header_end, sql_start); + } + let header_start_chars = text[..header_start].chars().count(); + let header_end_chars = header_start_chars + text[header_start..header_end].chars().count(); + let sql_start_chars = header_end_chars + text[header_end..sql_start].chars().count(); + (header_start_chars, header_end_chars, sql_start_chars) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs index 95639bca96..2c80363b66 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod matching; pub(crate) mod tokenization; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs index 761d058af1..4059a51c27 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod model_header_matching; pub(crate) mod model_header_parsing; pub(crate) mod model_header_tokenizing; pub(crate) mod sql_interpolation; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs b/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs new file mode 100644 index 0000000000..bbe71611bd --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs @@ -0,0 +1,8 @@ +use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; + +pub(crate) fn match_batch(contents: &[String]) -> Vec> { + contents + .iter() + .map(|text| matching::match_one(text)) + .collect() +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests.rs b/crates/sqlbuild-rules-native/src/compiler/tests.rs index 1a3a7a928d..85fc3262d1 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_header_matching.rs"] +mod model_header_matching; #[path = "tests/test_model_header_tokenization.rs"] mod model_header_tokenization; #[path = "tests/test_sql_interpolation.rs"] diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs new file mode 100644 index 0000000000..0b99b03c5e --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs @@ -0,0 +1,72 @@ +use crate::compiler::main::model_header_matching::match_batch; +use crate::compiler::tests::test_types::ModelHeaderMatchTestCase; + +#[test] +fn given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern() { + let test_cases = [ + ModelHeaderMatchTestCase { + description: "simple header", + contents: "MODEL (name orders);\nSELECT 1", + expected_offsets: Some((7, 18, 21)), + }, + ModelHeaderMatchTestCase { + description: "leading whitespace and spaced terminator", + contents: " \n MODEL\t( a ) \n ; \n SELECT 1", + expected_offsets: Some((10, 13, 21)), + }, + ModelHeaderMatchTestCase { + description: "closing parenthesis not followed by a semicolon stays in the header", + contents: "MODEL (columns (id INT));SELECT 1", + expected_offsets: Some((7, 23, 25)), + }, + ModelHeaderMatchTestCase { + description: "quoted closing parenthesis and semicolon stay in the header", + contents: "MODEL (description 'a);b');SELECT 1", + expected_offsets: Some((7, 25, 27)), + }, + ModelHeaderMatchTestCase { + description: "unclosed quote runs to the next parenthesis", + contents: "MODEL (description 'open);SELECT 1", + expected_offsets: Some((7, 24, 26)), + }, + ModelHeaderMatchTestCase { + description: "escaped quote inside a string", + contents: "MODEL (description \"a\\\"b\");SELECT 1", + expected_offsets: Some((7, 25, 27)), + }, + ModelHeaderMatchTestCase { + description: "non ascii offsets are code points", + contents: "MODEL (description 'caf\u{e9}')\u{a0};\u{3000}SELECT 1", + expected_offsets: Some((7, 25, 29)), + }, + ModelHeaderMatchTestCase { + description: "missing terminator", + contents: "MODEL (name orders)\nSELECT 1", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "trailing lone backslash", + contents: "MODEL (name orders\\", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "lowercase keyword", + contents: "model (name orders);", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "empty file", + contents: "", + expected_offsets: None, + }, + ]; + + for test_case in test_cases { + let actual = match_batch(&[test_case.contents.to_string()])[0]; + assert_eq!( + actual, test_case.expected_offsets, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs index dff917362b..bf566c43ff 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs @@ -86,3 +86,9 @@ pub(crate) struct RelationMarkerOracleTestCase { pub(crate) dialect: &'static str, pub(crate) expected_mismatches: &'static [&'static str], } + +pub(crate) struct ModelHeaderMatchTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) expected_offsets: Option<(usize, usize, usize)>, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d90c492dac..e132ef6d4d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -31,6 +31,7 @@ def load_config_json(project_dir: str) -> str: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... +def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index d8026940b2..d2e37bec65 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -34,6 +34,7 @@ build_integration_loader_functions, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, DiscoveredAssetFunction, @@ -81,9 +82,12 @@ def build_discovered_project_inputs( ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" - with OperationLifecycle( - operation_kind="project", operation_name="discovery_declaration_parse" - ) as declaration_lifecycle: + with ( + OperationLifecycle( + operation_kind="project", operation_name="discovery_declaration_parse" + ) as declaration_lifecycle, + DirectorySnapshot.scope(project_dir=project_dir), + ): source_files: tuple[DiscoveredSourceFile, ...] = discover_source_files( project_dir=project_dir, fact_cache=fact_cache ) @@ -227,6 +231,11 @@ def build_discovered_project_inputs( def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" + with DirectorySnapshot.scope(project_dir=project_dir): + return _build_tolerant_scope_discovery(project_dir=project_dir) + + +def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: project_config, local_config, config_faults = _discover_configs(project_dir=project_dir) models, model_faults = _discover_models(project_dir=project_dir) enums, constants, macros, declaration_faults = _discover_declarations(project_dir=project_dir) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 0abb5c741f..30f325738c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -5,14 +5,13 @@ import ast import importlib.util import inspect -import re import sys from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass, field from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import get_type_hints +from typing import cast, get_type_hints from pydantic import ValidationError @@ -26,6 +25,10 @@ named_declaration_files, named_declaration_roots, ) +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( + is_in_scoped_declaration_tree, + project_relative_path, +) from sqlbuild.compiler.discovery._helpers.filesystem.sql_test_files import discover_sql_test_files from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file @@ -37,18 +40,18 @@ from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - match_model_header, + match_model_headers, prepare_matched_model_file_headers, ) from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SCHEMA_FILE_NAME, SEED_FILE_SUFFIX, - SQL_TESTS_OWNERSHIP_ROOT, YAML_FILE_SUFFIXES, ) from sqlbuild.compiler.discovery.exceptions import ( @@ -89,6 +92,7 @@ DiscoveredSqlTestFile, DiscoveredTaskFunction, DiscoveryFileFault, + ModelHeaderMatch, NamedDeclarationRoot, ) from sqlbuild.compiler.discovery.types import ScopedDeclarationFile @@ -101,8 +105,8 @@ DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, GROUPED_NAMED_DECLARATION_DIRECTORIES, - INHERITED_DECLARATION_DIRECTORIES, LOCAL_DECLARATION_DIRECTORIES, + SCOPED_DECLARATION_DIRECTORIES, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.provider.classes.provider import Provider @@ -135,6 +139,7 @@ from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" +_FACTS_MEMO_KEY: str = "declaration_file_facts" @dataclass @@ -172,18 +177,25 @@ class _DeclarationFileFacts: declaration_root: Path -_SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( - INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES -) -_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" -_SQL_FILE_SUFFIX: str = ".sql" -_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts - - def _discover_declaration_file_facts( *, project_dir: Path, declaration_kind: DeclarationKind | None = None ) -> tuple[_DeclarationFileFacts, ...]: - _validate_declaration_groups(project_dir=project_dir) + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + memo_key: tuple[str, DeclarationKind | None] = (_FACTS_MEMO_KEY, declaration_kind) + cached: object = tree.memo.get(memo_key) + if cached is not None: + return cast(tuple[_DeclarationFileFacts, ...], cached) + facts: tuple[_DeclarationFileFacts, ...] = _scan_declaration_file_facts( + project_dir=project_dir, declaration_kind=declaration_kind, tree=tree + ) + tree.memo[memo_key] = facts + return facts + + +def _scan_declaration_file_facts( + *, project_dir: Path, declaration_kind: DeclarationKind | None, tree: DirectorySnapshot +) -> tuple[_DeclarationFileFacts, ...]: + _validate_declaration_groups(project_dir=project_dir, tree=tree) for directory_name in sorted(LOCAL_DECLARATION_DIRECTORIES): directory_kind, _scope_kind = DECLARATION_DIRECTORY_FACTS[directory_name] if (declaration_kind is None or directory_kind is declaration_kind) and ( @@ -204,6 +216,7 @@ def _discover_declaration_file_facts( facts.extend( _declaration_files_under_root( project_dir=project_dir, + tree=tree, ownership_root=Path(global_directory), declaration_root=declaration_root, owning_path=None, @@ -217,17 +230,17 @@ def _discover_declaration_file_facts( if not authored_root.is_dir(): continue directory: Path - for directory in sorted(path for path in authored_root.rglob("*") if path.is_dir()): - if directory.name not in _SCOPED_DECLARATION_DIRECTORIES: + for directory in sorted(tree.directories(root=authored_root)): + if directory.name not in SCOPED_DECLARATION_DIRECTORIES: continue directory_kind, directory_scope_kind = DECLARATION_DIRECTORY_FACTS[directory.name] if declaration_kind is not None and directory_kind is not declaration_kind: continue - relative_directory: Path = _project_relative_path( + relative_directory: Path = project_relative_path( path=directory, project_dir=project_dir ) descendants: tuple[str, ...] = relative_directory.parts[len(root_components) :] - if any(part in _SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): + if any(part in SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): raise DeclarationParseError( f"Declaration root {relative_directory.as_posix()}/ is nested inside another " "declaration tree" @@ -238,6 +251,7 @@ def _discover_declaration_file_facts( facts.extend( _declaration_files_under_root( project_dir=project_dir, + tree=tree, ownership_root=Path(*root_components), declaration_root=directory, owning_path=owning_path, @@ -247,7 +261,7 @@ def _discover_declaration_file_facts( return tuple(sorted(facts, key=lambda item: item.relative_path.as_posix())) -def _validate_declaration_groups(*, project_dir: Path) -> None: +def _validate_declaration_groups(*, project_dir: Path, tree: DirectorySnapshot) -> None: root_group: Path = project_dir / DECLARATION_GROUP_DIRECTORY if root_group.exists(): raise DeclarationParseError( @@ -259,7 +273,9 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: if not authored_root.is_dir(): continue for group in sorted( - path for path in authored_root.rglob(DECLARATION_GROUP_DIRECTORY) if path.is_dir() + path + for path in tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY) + if path.is_dir() ): if group.parent == authored_root: raise DeclarationParseError( @@ -273,12 +289,12 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: for child in group.iterdir() if not child.is_dir() or child.name - not in _SCOPED_DECLARATION_DIRECTORIES | GROUPED_NAMED_DECLARATION_DIRECTORIES + not in SCOPED_DECLARATION_DIRECTORIES | GROUPED_NAMED_DECLARATION_DIRECTORIES ) ) if unsupported: rendered: str = ", ".join( - _project_relative_path(path=path, project_dir=project_dir).as_posix() + project_relative_path(path=path, project_dir=project_dir).as_posix() for path in unsupported ) raise DeclarationParseError( @@ -290,6 +306,7 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: def _declaration_files_under_root( *, project_dir: Path, + tree: DirectorySnapshot, ownership_root: Path, declaration_root: Path, owning_path: Path | None, @@ -298,10 +315,10 @@ def _declaration_files_under_root( nested_root: Path for nested_root in sorted( path - for path in declaration_root.rglob("*") - if path.is_dir() and path.name in _SCOPED_DECLARATION_DIRECTORIES + for path in tree.directories(root=declaration_root) + if path.name in SCOPED_DECLARATION_DIRECTORIES ): - relative_nested_root: str = _project_relative_path( + relative_nested_root: str = project_relative_path( path=nested_root, project_dir=project_dir ).as_posix() raise DeclarationParseError( @@ -311,18 +328,18 @@ def _declaration_files_under_root( suffix: str = ".py" if declaration_kind is DeclarationKind.MACRO else ".sql" results: list[_DeclarationFileFacts] = [] file_path: Path - for file_path in sorted(declaration_root.rglob(f"*{suffix}")): + for file_path in sorted(tree.rglob(root=declaration_root, pattern=f"*{suffix}")): if declaration_kind is DeclarationKind.MACRO and file_path.stem == PYTHON_INIT_MODULE_STEM: continue results.append( _DeclarationFileFacts( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), declaration_kind=declaration_kind, scope_kind=scope_kind, ownership_root=ownership_root, owning_path=owning_path, - declaration_root=_project_relative_path( + declaration_root=project_relative_path( path=declaration_root, project_dir=project_dir ), ) @@ -330,28 +347,6 @@ def _declaration_files_under_root( return results -def _is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: - relative_parts: tuple[str, ...] = _project_relative_path( - path=file_path, project_dir=project_dir - ).parts - root_components: tuple[str, ...] - for root_components in CANONICAL_AUTHORED_ROOTS: - if relative_parts[: len(root_components)] != root_components: - continue - scoped_components: tuple[str, ...] = relative_parts[len(root_components) : -1] - if ( - root_components == _SQL_TEST_ROOT_COMPONENTS - and file_path.suffix == _SQL_FILE_SUFFIX - and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) - ): - return False - return any( - component == DECLARATION_GROUP_DIRECTORY or component in _SCOPED_DECLARATION_DIRECTORIES - for component in scoped_components - ) - return False - - def discover_model_files( *, project_dir: Path, @@ -368,25 +363,33 @@ def discover_model_files( loaded_model_files: list[tuple[Path, str | None, Exception | None]] = [] file_path: Path - for file_path in sorted(model_root.rglob("*.sql")): + for file_path in _sorted_glob(project_dir=project_dir, root=model_root, pattern="*.sql"): if selected_model_names is not None and file_path.stem not in selected_model_names: continue - if _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): + if is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): continue try: loaded_model_files.append((file_path, file_path.read_text(encoding="utf-8"), None)) except (OSError, UnicodeError, ValueError, SyntaxError) as error: loaded_model_files.append((file_path, None, error)) - header_matches: list[re.Match[str] | None] = [ - match_model_header(contents) if contents is not None and error is None else None + readable_contents: list[str] = [ + contents + for _path, contents, error in loaded_model_files + if contents is not None and error is None + ] + readable_matches: Iterator[ModelHeaderMatch | None] = iter( + match_model_headers(readable_contents) + ) + header_matches: list[ModelHeaderMatch | None] = [ + next(readable_matches) if contents is not None and error is None else None for _path, contents, error in loaded_model_files ] prepare_matched_model_file_headers(header_matches) discovered_model_files: list[DiscoveredSqlModelFile] = [] contents: str | None read_error: Exception | None - header_match: re.Match[str] | None + header_match: ModelHeaderMatch | None for (file_path, contents, read_error), header_match in zip( loaded_model_files, header_matches, strict=True ): @@ -401,7 +404,7 @@ def discover_model_files( discovered_model_files.append( discover_matched_model_file( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_match=header_match, extract_implicit_alias_columns=extract_implicit_alias_columns, @@ -535,8 +538,8 @@ def _parse_discovered_paths[DiscoveredT]( def _unscoped_files(*, root: Path, pattern: str, project_dir: Path) -> Iterator[Path]: file_path: Path - for file_path in sorted(root.rglob(pattern)): - if not _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): + for file_path in _sorted_glob(project_dir=project_dir, root=root, pattern=pattern): + if not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): yield file_path @@ -548,7 +551,7 @@ def discover_model_schema_files( def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredModelSchemaFile: file_path: Path = item[1] contents: str = file_path.read_text(encoding="utf-8") - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) return DiscoveredModelSchemaFile( file_path=file_path, relative_path=relative_path, @@ -581,6 +584,7 @@ def _parse_named_declaration_files[DiscoveredT: ScopedDeclarationFile]( items=( item for item in named_declaration_files( + project_dir=project_dir, roots=named_declaration_roots(project_dir=project_dir, kinds=kinds), pattern="*.sql", ) @@ -608,7 +612,7 @@ def parse(file_path: Path) -> DiscoveredSqlFunctionFile: header_values, body_sql = parse_function_sql(contents=contents, file_path=file_path) return DiscoveredSqlFunctionFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, body_sql=body_sql, @@ -633,7 +637,7 @@ def discover_python_function_files( discovered_function_files: list[DiscoveredPythonFunctionFile] = [] file_path: Path - for file_path in sorted(function_root.rglob("*.py")): + for file_path in _sorted_glob(project_dir=project_dir, root=function_root, pattern="*.py"): contents: str = file_path.read_text(encoding="utf-8") header_values: dict[str, object] entry_point: str @@ -644,7 +648,7 @@ def discover_python_function_files( discovered_function_files.append( DiscoveredPythonFunctionFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, entry_point=entry_point, @@ -657,6 +661,7 @@ def discover_python_function_files( def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: """Discover model schema.yml files and seed declaration .yml files.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) schema_paths: list[Path] = [] models_root: Path = project_dir / "models" seeds_root: Path = project_dir / "seeds" @@ -664,17 +669,17 @@ def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, . if models_root.is_dir(): schema_paths.extend( path - for path in sorted(models_root.rglob(SCHEMA_FILE_NAME)) - if not _is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) + for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) + if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) ) if seeds_root.is_dir(): yaml_path: Path - for yaml_path in sorted(seeds_root.rglob("*.yaml")): + for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): raise SchemaParseError( f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " ".yaml is not supported" ) - schema_paths.extend(sorted(seeds_root.rglob("*.yml"))) + schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) deduped_paths: tuple[Path, ...] = tuple(dict.fromkeys(schema_paths)) discovered_schema_files: list[DiscoveredSchemaFile] = [] @@ -687,7 +692,7 @@ def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, . discovered_schema_files.append( DiscoveredSchemaFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, model_entries=model_entries, seed_entries=seed_entries, @@ -716,7 +721,7 @@ def parse(file_path: Path) -> DiscoveredSourceFile: return parse_source_file_with_cache( project_dir=project_dir, file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), fact_cache=fact_cache, ) @@ -735,9 +740,9 @@ def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: return tuple( DiscoveredSeedFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) - for file_path in sorted(seeds_root.rglob("*")) + for file_path in _sorted_glob(project_dir=project_dir, root=seeds_root, pattern="*") if file_path.is_file() and file_path.suffix == SEED_FILE_SUFFIX ) @@ -757,9 +762,9 @@ def discover_test_files( file_paths: tuple[Path, ...] = tuple( file_path - for file_path in sorted(tests_root.rglob("*.sql")) + for file_path in _sorted_glob(project_dir=project_dir, root=tests_root, pattern="*.sql") if (selected_paths is None or file_path.resolve() in selected_paths) - and not _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir) + and not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir) ) return discover_sql_test_files( project_dir=project_dir, @@ -782,7 +787,7 @@ def parse(file_path: Path) -> DiscoveredSqlScenarioFile: return parse_sql_scenario_file( contents=file_path.read_text(encoding="utf-8"), file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) return _parse_discovered_paths( @@ -803,7 +808,7 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredAuditFile: contents: str = file_path.read_text(encoding="utf-8") return DiscoveredAuditFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, blocks=parse_sql_audit_file(contents=contents, file_path=file_path), declaration_kind=root.kind, @@ -842,7 +847,7 @@ def discover_macro_files( def _discovery_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault: try: - relative_path: Path | None = _project_relative_path(path=path, project_dir=project_dir) + relative_path: Path | None = project_relative_path(path=path, project_dir=project_dir) except ValueError: relative_path = None message: str = str(error).replace(str(project_dir), ".") @@ -871,7 +876,7 @@ def discover_materialization_files( discovered_files.append( DiscoveredMaterializationFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=file_path.stem, provider_usages=( _provider_usages(function=materialize_fn, provider_by_name=provider_by_name) @@ -912,6 +917,7 @@ def discover_hook_functions( root: NamedDeclarationRoot file_path: Path for root, file_path in named_declaration_files( + project_dir=project_dir, roots=named_declaration_roots( project_dir=project_dir, kinds=frozenset({DeclarationKind.PYTHON_HOOK}) ), @@ -941,7 +947,7 @@ def discover_hook_functions( root.place( DiscoveredHookFunction( file_path=file_path, - relative_path=_project_relative_path( + relative_path=project_relative_path( path=file_path, project_dir=project_dir ), name=hook_definition.name, @@ -968,7 +974,7 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredSqlHookFile: return parse_sql_hook_file( contents=file_path.read_text(encoding="utf-8"), file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) return _parse_named_declaration_files( @@ -1028,7 +1034,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, discovered_providers.append( DiscoveredProvider( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=provider_name, provider_class=provider_class, settings=_provider_instance( @@ -1117,7 +1123,7 @@ def discover_event_exporter_declarations( discovered.append( DiscoveredEventExporterDeclaration( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=definition.name, function=value, event_kinds=definition.event_kinds, @@ -1166,7 +1172,7 @@ def _validate_event_exporter_declaration_signature( file_path: Path, project_dir: Path, ) -> None: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if inspect.iscoroutinefunction(function): raise EventExporterDiscoveryError( f"Event exporter '{exporter_name}' in {relative_path} must be synchronous" @@ -1222,7 +1228,7 @@ def _bind_event_exporter_provider_usages( project_dir: Path, provider_by_name: dict[str, DiscoveredProvider], ) -> tuple[DiscoveredProviderUsage, ...]: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) parameters: tuple[inspect.Parameter, ...] = tuple( inspect.signature(function).parameters.values() ) @@ -1257,7 +1263,7 @@ def _public_python_files(*, root: Path) -> tuple[Path, ...]: def _provider_name(*, provider_class: type[Provider], file_path: Path, project_dir: Path) -> str: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) explicit_name: str | None = provider_class.provider_name if explicit_name is not None: validate_resource_identity( @@ -1286,7 +1292,7 @@ def _provider_instance( try: return provider_class() except ValidationError as error: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) raise ProviderDiscoveryError( f"Provider '{provider_name}' in {relative_path} has invalid settings:\n" f"{_format_provider_validation_error(error)}" @@ -1458,7 +1464,7 @@ def _restore_python_root_modules(*, saved_modules: dict[str, ModuleType]) -> Non def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: """Import a python/ module under its package name, reusing it within a discovery pass.""" - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) module_name: str = ".".join(relative_path.with_suffix("").parts) old_path: list[str] = list(sys.path) sys.path.insert(0, str(project_dir)) @@ -1556,7 +1562,7 @@ def _append_module_audit_factories( definition: AuditFactoryDefinition | None = read_audit_factory_definition(value) if definition is None: continue - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if _python_node_definition_names(value): kinds: str = ", ".join(_python_node_definition_names(value)) raise PythonNodeDiscoveryError( @@ -1607,7 +1613,7 @@ def _call_audit_factory( file_path: Path, project_dir: Path, ) -> tuple[AuditCase, ...]: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if inspect.signature(factory).parameters: raise PythonNodeDiscoveryError( f"Audit factory '{definition.name}' in {relative_path} must not require arguments" @@ -1648,7 +1654,7 @@ def _append_python_node_function( bucket.add_loader( DiscoveredLoaderFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=loader_definition.name, function=function, depends_on=loader_definition.depends_on, @@ -1671,7 +1677,7 @@ def _append_python_node_function( bucket.add_task( DiscoveredTaskFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=task_definition.name, function=function, depends_on=task_definition.depends_on, @@ -1692,7 +1698,7 @@ def _append_python_node_function( bucket.add_asset( DiscoveredAssetFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=asset_definition.name, function=function, depends_on=asset_definition.depends_on, @@ -1715,7 +1721,7 @@ def _append_python_node_function( bucket.add_check( DiscoveredCheckFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=check_definition.name, function=function, depends_on=check_definition.depends_on, @@ -1795,7 +1801,7 @@ def _load_python_node_module(*, file_path: Path, project_dir: Path) -> ModuleTyp return _exec_project_module( module_name="sqlbuild_project_python_node_" + "_".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ), file_path=file_path, project_dir=project_dir, @@ -1830,7 +1836,7 @@ def _load_project_package_module( error_type: type[Exception], ) -> ModuleType: module_name: str = ".".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ) _evict_stale_project_package_modules( root_module=module_name.split(".", maxsplit=1)[0], @@ -1921,7 +1927,7 @@ def _load_materialization_module(*, file_path: Path, project_dir: Path) -> Modul raise PythonNodeDiscoveryError(removed_hook_message) module: ModuleType = _exec_project_module( module_name=".".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ), file_path=file_path, project_dir=project_dir, @@ -1942,21 +1948,16 @@ def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: return DiscoveredAdapterFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) -def _project_relative_path(*, path: Path, project_dir: Path) -> Path: - """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" +def _sorted_glob(*, project_dir: Path, root: Path, pattern: str) -> list[Path]: + """Return ``sorted(root.rglob(pattern))`` from the active discovery snapshot.""" - root_parts: tuple[str, ...] = project_dir.parts - path_parts: tuple[str, ...] = path.parts - if ( - path.is_absolute() is not project_dir.is_absolute() - or path_parts[: len(root_parts)] != root_parts - ): - return path.relative_to(project_dir) - return path.with_segments(*path_parts[len(root_parts) :]) + return sorted( + DirectorySnapshot.current(project_dir=project_dir).rglob(root=root, pattern=pattern) + ) def _is_relative_to(*, path: Path, parent: Path) -> bool: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py index 701ce4b7e7..7a31694a5c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from pathlib import Path from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -14,7 +13,7 @@ matched_model_output_column_locations, parse_matched_model_sql, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, ModelHeaderMatch def discover_matched_model_file( @@ -22,7 +21,7 @@ def discover_matched_model_file( file_path: Path, relative_path: Path, contents: str, - header_match: re.Match[str] | None, + header_match: ModelHeaderMatch | None, extract_implicit_alias_columns: bool, extract_output_column_locations: bool, ) -> DiscoveredSqlModelFile: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index 82ed8d592a..9d0156d070 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -4,7 +4,9 @@ from collections.abc import Iterator from pathlib import Path +from typing import cast +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import CANONICAL_AUTHORED_ROOTS from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import NamedDeclarationRoot @@ -21,6 +23,8 @@ _AUDIT_ROLE_DIRECTORY: str = "audits" _LOCAL_AUDIT_ROLE_DIRECTORY: str = "_audits" _SINGULAR_AUDIT_DIRECTORY: str = "singular" +_LAYOUT_MEMO_KEY: str = "named_declaration_layout" +_GROUPS_MEMO_KEY: str = "declaration_groups" _NESTED_ROLE_DIRECTORIES: frozenset[str] = frozenset( {DECLARATION_GROUP_DIRECTORY} | INHERITED_DECLARATION_DIRECTORIES @@ -33,7 +37,10 @@ def named_declaration_roots( ) -> tuple[NamedDeclarationRoot, ...]: """Return validated top-level and grouped declaration roots for the requested kinds.""" - validate_named_declaration_layout(project_dir=project_dir) + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + if _LAYOUT_MEMO_KEY not in tree.memo: + validate_named_declaration_layout(project_dir=project_dir) + tree.memo[_LAYOUT_MEMO_KEY] = True roots: list[NamedDeclarationRoot] = [] for role_parts, kind in GLOBAL_NAMED_DECLARATION_ROOTS.items(): directory: Path = project_dir.joinpath(*role_parts) @@ -67,20 +74,22 @@ def named_declaration_roots( def named_declaration_files( - *, roots: tuple[NamedDeclarationRoot, ...], pattern: str + *, project_dir: Path, roots: tuple[NamedDeclarationRoot, ...], pattern: str ) -> Iterator[tuple[NamedDeclarationRoot, Path]]: """Yield every matching file below each root, paired with its root.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) root: NamedDeclarationRoot for root in roots: file_path: Path - for file_path in sorted(root.directory.rglob(pattern)): + for file_path in sorted(tree.rglob(root=root.directory, pattern=pattern)): yield root, file_path def validate_named_declaration_layout(*, project_dir: Path) -> None: """Reject entries that no named declaration role accepts.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) audits_root: Path = project_dir / _AUDIT_ROLE_DIRECTORY if audits_root.is_dir(): _require_role_children( @@ -93,7 +102,7 @@ def validate_named_declaration_layout(*, project_dir: Path) -> None: directory: Path = project_dir / top_level if not directory.is_dir(): continue - for nested in sorted(path for path in directory.rglob("*") if path.is_dir()): + for nested in sorted(tree.directories(root=directory)): if nested.name in _NESTED_ROLE_DIRECTORIES: nested_path: str = _relative(path=nested, project_dir=project_dir) raise DeclarationParseError( @@ -154,6 +163,10 @@ def _require_role_children( def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Path], ...]: + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + cached: object = tree.memo.get(_GROUPS_MEMO_KEY) + if cached is not None: + return cast(tuple[tuple[tuple[str, ...], Path], ...], cached) groups: list[tuple[tuple[str, ...], Path]] = [] for root_components in CANONICAL_AUTHORED_ROOTS: authored_root: Path = project_dir.joinpath(*root_components) @@ -161,10 +174,12 @@ def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Pa continue groups.extend( (root_components, group) - for group in sorted(authored_root.rglob(DECLARATION_GROUP_DIRECTORY)) + for group in sorted(tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY)) if group.is_dir() and group.parent != authored_root ) - return tuple(groups) + result: tuple[tuple[tuple[str, ...], Path], ...] = tuple(groups) + tree.memo[_GROUPS_MEMO_KEY] = result + return result def _relative(*, path: Path, project_dir: Path) -> str: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py new file mode 100644 index 0000000000..0db80edd8a --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py @@ -0,0 +1,68 @@ +"""Project-relative path facts shared by filesystem discovery.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from sqlbuild.compiler.discovery.constants import ( + CANONICAL_AUTHORED_ROOTS, + SQL_TESTS_OWNERSHIP_ROOT, +) +from sqlbuild.compiler.scopes.constants import ( + DECLARATION_GROUP_DIRECTORY, + SCOPED_DECLARATION_DIRECTORIES, +) +from sqlbuild.compiler.scopes.types import DeclarationKind + +_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" +_SQL_FILE_SUFFIX: str = ".sql" +_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts + + +def project_relative_path(*, path: Path, project_dir: Path) -> Path: + """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" + + root_parts: tuple[str, ...] = project_dir.parts + path_parts: tuple[str, ...] = path.parts + if ( + path.is_absolute() is not project_dir.is_absolute() + or path_parts[: len(root_parts)] != root_parts + ): + return path.relative_to(project_dir) + return path.with_segments(*path_parts[len(root_parts) :]) + + +def is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: + """Return whether a file lives inside a scoped declaration tree rather than a resource tree.""" + + return _is_scoped_declaration_directory( + directory=file_path.parent, + project_dir=project_dir, + sql_file=file_path.suffix == _SQL_FILE_SUFFIX, + ) + + +@lru_cache(maxsize=8192) +def _is_scoped_declaration_directory(*, directory: Path, project_dir: Path, sql_file: bool) -> bool: + """Classify a file's directory once; every file in it shares the same scoped-tree answer.""" + + directory_parts: tuple[str, ...] = project_relative_path( + path=directory, project_dir=project_dir + ).parts + root_components: tuple[str, ...] + for root_components in CANONICAL_AUTHORED_ROOTS: + if directory_parts[: len(root_components)] != root_components: + continue + scoped_components: tuple[str, ...] = directory_parts[len(root_components) :] + if ( + root_components == _SQL_TEST_ROOT_COMPONENTS + and sql_file + and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) + ): + return False + return any( + component == DECLARATION_GROUP_DIRECTORY or component in SCOPED_DECLARATION_DIRECTORIES + for component in scoped_components + ) + return False diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 117965b523..e9876628a6 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -23,6 +23,7 @@ ) from sqlbuild.compiler.discovery.models import ( ModelHeaderColumnSpan, + ModelHeaderMatch, NamedSqlHookEntry, PythonHookEntry, SqlHookEntry, @@ -115,10 +116,34 @@ def put(self, *, header: str, tokenization: _ModelHeaderTokenization) -> None: _MODEL_HEADER_TOKEN_CACHE: _ModelHeaderTokenCache = _ModelHeaderTokenCache() -def match_model_header(contents: str) -> re.Match[str] | None: +def match_model_header(contents: str) -> ModelHeaderMatch | None: """Match the MODEL(...) header once so callers can reuse it for every header projection.""" - return _MODEL_HEADER_PATTERN.match(contents) + header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) + if header_match is None: + return None + return ModelHeaderMatch( + contents=contents, + header_start=header_match.start("header"), + header_end=header_match.end("header"), + sql_start=header_match.start("sql"), + ) + + +def match_model_headers(contents: list[str]) -> list[ModelHeaderMatch | None]: + """Match many MODEL(...) headers natively with the same semantics as ``match_model_header``.""" + + return [ + None + if offsets is None + else ModelHeaderMatch( + contents=value, + header_start=offsets[0], + header_end=offsets[1], + sql_start=offsets[2], + ) + for value, offsets in zip(contents, _native.match_model_headers(contents), strict=True) + ] def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: @@ -128,7 +153,7 @@ def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object def parse_matched_model_sql( - *, header_match: re.Match[str] | None, file_path: Path + *, header_match: ModelHeaderMatch | None, file_path: Path ) -> tuple[dict[str, object], str]: """Parse header values and SQL body from one precomputed MODEL header match.""" @@ -139,7 +164,7 @@ def parse_matched_model_sql( ) header_values: dict[str, object] = parse_header_values( - header=header_match.group("header"), + header=header_match.header, file_path=file_path, statement_name="MODEL", ) @@ -153,12 +178,12 @@ def parse_matched_model_sql( header_values=header_values, supported_keys=SQL_MODEL_HEADER_KEYS, statement="MODEL()", - header=header_match.group("header"), - header_line=header_match.string.count("\n", 0, header_match.start("header")) + 1, + header=header_match.header, + header_line=header_match.contents.count("\n", 0, header_match.header_start) + 1, file_path=file_path, error_class=ModelSqlParseError, ) - query: str = header_match.group("sql").strip() + query: str = header_match.sql.strip() if not query: raise ModelSqlParseError(f"SQL model '{file_path}' must contain SQL after MODEL(...)") return header_values, query @@ -175,7 +200,7 @@ def model_header_column_locations( def matched_model_header_column_locations( - *, contents: str, header_match: re.Match[str] | None, relative_path: Path + *, contents: str, header_match: ModelHeaderMatch | None, relative_path: Path ) -> dict[str, SourceLocation]: """Return authored MODEL(columns) locations from one precomputed header match.""" @@ -183,8 +208,8 @@ def matched_model_header_column_locations( return {} return header_column_locations( contents=contents, - header=header_match.group("header"), - header_start=header_match.start("header"), + header=header_match.header, + header_start=header_match.header_start, relative_path=relative_path, ) @@ -314,7 +339,7 @@ def model_output_column_locations( def matched_model_output_column_locations( *, contents: str, - header_match: re.Match[str] | None, + header_match: ModelHeaderMatch | None, relative_path: Path, extract_implicit_alias_columns: bool, ) -> dict[str, SourceLocation]: @@ -322,8 +347,8 @@ def matched_model_output_column_locations( if header_match is None: return {} - sql_start: int = header_match.start("sql") - sql: str = header_match.group("sql") + sql_start: int = header_match.sql_start + sql: str = header_match.sql projection_ranges: tuple[tuple[int, int], ...] | None = _top_level_select_projection_ranges(sql) if projection_ranges is None: return {} @@ -674,15 +699,11 @@ def prepare_model_file_headers(contents: list[str]) -> None: prepare_matched_model_file_headers([match_model_header(value) for value in contents]) -def prepare_matched_model_file_headers(header_matches: list[re.Match[str] | None]) -> None: +def prepare_matched_model_file_headers(header_matches: list[ModelHeaderMatch | None]) -> None: """Batch tokenization for precomputed MODEL header matches.""" prepare_model_header_tokens( - [ - header_match.group("header") - for header_match in header_matches - if header_match is not None - ] + [header_match.header for header_match in header_matches if header_match is not None] ) diff --git a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py new file mode 100644 index 0000000000..fe030ff14d --- /dev/null +++ b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py @@ -0,0 +1,135 @@ +"""One shared directory snapshot for a discovery pass.""" + +from __future__ import annotations + +import fnmatch +import os +import re +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from functools import lru_cache +from pathlib import Path +from typing import ClassVar + +from sqlbuild.compiler.discovery.types import DirectorySnapshotEntry + +_CASE_PROBE: str = "A" +_DIRECTORIES_QUERY: str = "" + + +class DirectorySnapshot: + """Directory listings read once per discovery pass and shared by every glob in it.""" + + active: ClassVar[ContextVar[DirectorySnapshot | None]] = ContextVar( + "sqlbuild_directory_snapshot", default=None + ) + + def __init__(self, *, project_dir: Path) -> None: + self.project_dir: Path = project_dir + self.memo: dict[object, object] = {} + self._listings: dict[Path, tuple[DirectorySnapshotEntry, ...]] = {} + self._matches: dict[tuple[Path, str], tuple[Path, ...]] = {} + + @classmethod + @contextmanager + def scope(cls, *, project_dir: Path) -> Iterator[DirectorySnapshot]: + """Share one snapshot across every discovery query for the project inside the scope.""" + + active: DirectorySnapshot | None = cls.active.get() + if active is not None and active.project_dir == project_dir: + yield active + return + tree: DirectorySnapshot = DirectorySnapshot(project_dir=project_dir) + token: Token[DirectorySnapshot | None] = cls.active.set(tree) + try: + yield tree + finally: + cls.active.reset(token) + + @classmethod + def current(cls, *, project_dir: Path) -> DirectorySnapshot: + """Return the active snapshot for the project, or a single-use one outside a scope.""" + + active: DirectorySnapshot | None = cls.active.get() + if active is not None and active.project_dir == project_dir: + return active + return DirectorySnapshot(project_dir=project_dir) + + def rglob(self, *, root: Path, pattern: str) -> tuple[Path, ...]: + """Return the paths ``root.rglob(pattern)`` yields for a single-name pattern, unordered.""" + + match: Callable[[str], bool] = _name_matcher(pattern) + return self._select(root=root, key=pattern, accept=lambda entry: match(entry.name)) + + def directories(self, *, root: Path) -> tuple[Path, ...]: + """Return every directory below ``root``, including unwalked directory links, unordered.""" + + return self._select(root=root, key=_DIRECTORIES_QUERY, accept=lambda entry: entry.is_dir) + + def _select( + self, *, root: Path, key: str, accept: Callable[[DirectorySnapshotEntry], bool] + ) -> tuple[Path, ...]: + cached: tuple[Path, ...] | None = self._matches.get((root, key)) + if cached is not None: + return cached + selected: list[Path] = [] + if root.is_dir(): + directory: Path + for directory in self._walk(root=root): + selected.extend( + directory / entry.name + for entry in self._listing(directory=directory) + if accept(entry) + ) + result: tuple[Path, ...] = tuple(selected) + self._matches[(root, key)] = result + return result + + def _walk(self, *, root: Path) -> Iterator[Path]: + pending: list[Path] = [root] + while pending: + directory: Path = pending.pop() + yield directory + pending.extend( + directory / entry.name + for entry in self._listing(directory=directory) + if entry.is_walkable_dir + ) + + def _listing(self, *, directory: Path) -> tuple[DirectorySnapshotEntry, ...]: + cached: tuple[DirectorySnapshotEntry, ...] | None = self._listings.get(directory) + if cached is not None: + return cached + entries: list[DirectorySnapshotEntry] = [] + try: + with os.scandir(directory) as scanned: + entries.extend( + DirectorySnapshotEntry( + name=entry.name, + is_dir=_is_dir(entry=entry, follow_symlinks=True), + is_walkable_dir=_is_dir(entry=entry, follow_symlinks=False), + ) + for entry in scanned + ) + except OSError: + entries = [] + listing: tuple[DirectorySnapshotEntry, ...] = tuple(entries) + self._listings[directory] = listing + return listing + + +def _is_dir(*, entry: os.DirEntry[str], follow_symlinks: bool) -> bool: + try: + return entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + return False + + +@lru_cache(maxsize=64) +def _name_matcher(pattern: str) -> Callable[[str], bool]: + """Match names like ``Path.glob`` with the platform's default case sensitivity.""" + + flags: int = 0 if os.path.normcase(_CASE_PROBE) == _CASE_PROBE else re.IGNORECASE + compiled: re.Pattern[str] = re.compile(fnmatch.translate(pattern), flags) + return lambda name: compiled.match(name) is not None diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index a43a6f99f9..68453d746e 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -81,6 +81,28 @@ class DiscoveredSqlModelFile: extract_implicit_alias_columns: bool = True +@dataclass(frozen=True) +class ModelHeaderMatch: + """Offsets of one matched ``MODEL(...)`` header within its file contents.""" + + contents: str + header_start: int + header_end: int + sql_start: int + + @property + def header(self) -> str: + """Return the header body between the MODEL parentheses.""" + + return self.contents[self.header_start : self.header_end] + + @property + def sql(self) -> str: + """Return everything after the header terminator and following whitespace.""" + + return self.contents[self.sql_start :] + + @dataclass(frozen=True) class ModelHeaderColumnSpan: """Authored byte offsets for one MODEL(columns) entry and metadata body.""" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index cdc21e7b20..fc1896e66c 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -5,7 +5,7 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import Any, ClassVar, Protocol +from typing import Any, ClassVar, NamedTuple, Protocol from sqlbuild.compiler.scopes.types import ScopeKind @@ -25,3 +25,11 @@ class ScopedDeclarationFile(Protocol): ownership_root: Path | None owning_path: Path | None declaration_root: Path | None + + +class DirectorySnapshotEntry(NamedTuple): + """One directory entry in a discovery snapshot.""" + + name: str + is_dir: bool + is_walkable_dir: bool diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index d57da707c8..0fdb41dd57 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -62,6 +62,9 @@ GLOBAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"macros", "enums", "constants"}) INHERITED_DECLARATION_DIRECTORIES: frozenset[str] = GLOBAL_DECLARATION_DIRECTORIES LOCAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"_macros", "_enums", "_constants"}) +SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( + INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES +) DECLARATION_DIRECTORY_FACTS: dict[str, tuple[DeclarationKind, ScopeKind]] = { "macros": (DeclarationKind.MACRO, ScopeKind.INHERITED), "enums": (DeclarationKind.ENUM, ScopeKind.INHERITED), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py new file mode 100644 index 0000000000..200495ee9a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -0,0 +1,217 @@ +"""Randomized trees, projects, and model files for shared-snapshot discovery tests.""" + +from __future__ import annotations + +import random +import re +from collections.abc import Callable +from itertools import compress +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem import aggregation +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_audit_files, + discover_constant_files, + discover_enum_files, + discover_macro_files, + discover_model_files, + discover_model_schema_files, + discover_schema_files, + discover_seed_files, + discover_sql_function_files, + discover_sql_hook_files, + discover_test_files, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import ( + match_model_header, + match_model_headers, +) +from sqlbuild.compiler.discovery._helpers.yml.project import ( + load_local_config, + load_project_config, +) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + +_DIRECTORY_NAMES: tuple[str, ...] = ( + "orders", + "customers", + "_sqlbuild", + "enums", + "_enums", + "macros", + ".hidden", +) +_FILE_NAMES: tuple[str, ...] = ( + "schema.yml", + "daily.sql", + "daily.SQL", + "helper.py", + "notes.yml", + ".draft.sql", +) +_LINK_NAMES: tuple[str, ...] = ("shared", "enums_link", "_sqlbuild_link") +_PATTERNS: tuple[str, ...] = ("*", "*.sql", "*.py", "*.yml", "schema.yml", "_sqlbuild", "enums") +_ISOLATED_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { + "model_files": discover_model_files, + "enum_files": discover_enum_files, + "constant_files": discover_constant_files, + "model_schema_files": discover_model_schema_files, + "sql_function_files": discover_sql_function_files, + "sql_hook_files": discover_sql_hook_files, + "schema_files": discover_schema_files, + "seed_files": discover_seed_files, + "test_files": discover_test_files, + "audit_files": discover_audit_files, + "macro_files": discover_macro_files, +} +_SCOPED_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "shop"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "models/sales/daily.sql": "MODEL (description 'Daily.');\nSELECT 1 AS day_id\n", + "models/sales/schema.yml": "models: []\n", + "models/sales/enums/region.sql": "ENUM (name region, members [NORTH, SOUTH]);\n", + "models/sales/_sqlbuild/constants/limit.sql": "CONSTANT (name order_limit, value 10);\n", + "models/sales/_sqlbuild/audits/generic/positive.sql": ( + "AUDIT (name positive);\nSELECT 1 WHERE FALSE\n" + ), + "macros/money.py": "def money(value):\n return value\n", + "enums/status.sql": "ENUM (name status, members [OPEN, CLOSED]);\n", + "functions/sql/tax.sql": ( + "FUNCTION (arguments (amount DOUBLE), returns DOUBLE);\namount * 0.2\n" + ), +} +_HEADER_FRAGMENTS: tuple[str, ...] = ( + "MODEL", + "model", + "(", + ")", + ";", + "'", + '"', + "\\", + "name orders", + "columns (id INT)", + "SELECT 1", + "\n", + " ", + "é", + "\u00a0", + "\u3000", + "\u2028", + "\x1c", + "\U0001f600", +) +_PYTHON_WHITESPACE: tuple[str, ...] = tuple(re.findall(r"\s", "".join(map(chr, range(0x3001))))) + + +def write_random_tree(*, root: Path, seed: int, entry_count: int) -> None: + """Write a seeded tree of folders, files, hidden entries, and directory links.""" + + rng: random.Random = random.Random(seed) + directories: list[Path] = [root] + creators: tuple[Callable[[random.Random, Path, list[Path]], None], ...] = ( + *(_add_directory,) * 9, + *(_add_file,) * 9, + *(_add_link,) * 2, + ) + for _index in range(entry_count): + rng.choice(creators)(rng, rng.choice(directories), directories) + + +def snapshot_glob_mismatches(*, root: Path) -> tuple[str, ...]: + """Return every root and pattern where the snapshot disagrees with ``Path.rglob``.""" + + tree: DirectorySnapshot = DirectorySnapshot(project_dir=root) + directories: list[Path] = [root, *sorted(filter(Path.is_dir, root.rglob("*")))] + labels: list[str] = [] + matches: list[bool] = [] + for directory in directories: + for pattern in _PATTERNS: + labels.append(f"{directory}:{pattern}") + matches.append( + sorted(tree.rglob(root=directory, pattern=pattern)) + == sorted(directory.rglob(pattern)) + ) + labels.append(f"{directory}:") + matches.append( + sorted(tree.directories(root=directory)) + == sorted(filter(Path.is_dir, directory.rglob("*"))) + ) + labels.append("absent root") + matches.append(tree.rglob(root=root / "absent", pattern="*") == ()) + return tuple(compress(labels, [not matched for matched in matches])) + + +def write_scoped_project(*, root: Path) -> None: + """Write a project with resources, scoped and grouped declarations, and global roles.""" + + for relative_path, contents in _SCOPED_PROJECT_FILES.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def isolated_discovery_mismatches(*, project_dir: Path) -> tuple[str, ...]: + """Return categories where one shared-snapshot pass differs from isolated discovery.""" + + inputs: DiscoveredProjectInputs = aggregation.build_discovered_project_inputs( + project_dir=project_dir, + project_config=load_project_config(project_dir=project_dir), + local_config=load_local_config(project_dir=project_dir), + sql_analysis_enabled=True, + ) + return tuple( + compress( + _ISOLATED_DISCOVERERS, + [ + getattr(inputs, name) != discover(project_dir=project_dir) + for name, discover in _ISOLATED_DISCOVERERS.items() + ], + ) + ) + + +def model_paths(*, inputs: DiscoveredProjectInputs) -> tuple[str, ...]: + """Return discovered model paths in discovery order.""" + + return tuple(file.relative_path.as_posix() for file in inputs.model_files) + + +def header_match_mismatches(*, seed: int, count: int) -> tuple[str, ...]: + """Return random MODEL files whose native header match differs from the Python pattern.""" + + rng: random.Random = random.Random(seed) + contents: list[str] = [_random_model_file(rng) for _ in range(count)] + contents.extend(f"MODEL (a){space};{space}SELECT 1" for space in _PYTHON_WHITESPACE) + return tuple( + compress( + contents, + [ + native != match_model_header(value) + for value, native in zip(contents, match_model_headers(contents), strict=True) + ], + ) + ) + + +def _random_model_file(rng: random.Random) -> str: + prefix: str = "".join(rng.choice(_PYTHON_WHITESPACE) for _ in range(rng.randint(0, 2))) + body: str = "".join(rng.choice(_HEADER_FRAGMENTS) for _ in range(rng.randint(0, 14))) + return f"{prefix}MODEL{rng.choice(_PYTHON_WHITESPACE)}({body}" + + +def _add_directory(rng: random.Random, parent: Path, directories: list[Path]) -> None: + path: Path = parent / rng.choice(_DIRECTORY_NAMES) + path.mkdir(exist_ok=True) + directories.append(path) + + +def _add_file(rng: random.Random, parent: Path, _directories: list[Path]) -> None: + (parent / rng.choice(_FILE_NAMES)).write_text("SELECT 1\n", encoding="utf-8") + + +def _add_link(rng: random.Random, parent: Path, directories: list[Path]) -> None: + link: Path = parent / rng.choice(_LINK_NAMES) + link.unlink(missing_ok=True) + link.symlink_to(rng.choice([*directories, parent / "missing"]), target_is_directory=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 0425fc326c..c85fe6b18e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -940,3 +940,42 @@ class StatementHeaderKeysTestCase: contents: str expected_error: str expected_help: str + + +@dataclass(frozen=True) +class SnapshotGlobTestCase: + description: str + seed: int + entry_count: int + expected_mismatches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SharedSnapshotDiscoveryTestCase: + description: str + expected_model_paths: tuple[str, ...] + expected_mismatches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class DiscoveryPassRefreshTestCase: + description: str + added_files: dict[str, str] + removed_path: str + expected_first_paths: tuple[str, ...] + expected_second_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class SnapshotScopeReuseTestCase: + description: str + expected_nested_reuse: bool + expected_later_reuse: bool + + +@dataclass(frozen=True) +class NativeHeaderMatchTestCase: + description: str + seed: int + count: int + expected_mismatches: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py new file mode 100644 index 0000000000..12c88af4de --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py @@ -0,0 +1,147 @@ +"""Tests for the shared single-walk discovery snapshot.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_helpers import ( + header_match_mismatches, + isolated_discovery_mismatches, + model_paths, + snapshot_glob_mismatches, + write_random_tree, + write_scoped_project, +) +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + DiscoveryPassRefreshTestCase, + NativeHeaderMatchTestCase, + SharedSnapshotDiscoveryTestCase, + SnapshotGlobTestCase, + SnapshotScopeReuseTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotGlobTestCase("small_tree", seed=1, entry_count=40), + SnapshotGlobTestCase("medium_tree", seed=2, entry_count=120), + SnapshotGlobTestCase("large_tree", seed=3, entry_count=300), + ], + ids=lambda case: case.description, +) +def test_given_randomized_tree_when_globbing_snapshot_then_matches_pathlib_rglob( + tmp_path: Path, test_case: SnapshotGlobTestCase +) -> None: + write_random_tree(root=tmp_path, seed=test_case.seed, entry_count=test_case.entry_count) + + mismatches: tuple[str, ...] = snapshot_glob_mismatches(root=tmp_path) + + assert mismatches == test_case.expected_mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + SharedSnapshotDiscoveryTestCase( + "scoped_and_grouped_declarations", + expected_model_paths=("models/orders.sql", "models/sales/daily.sql"), + ) + ], + ids=lambda case: case.description, +) +def test_given_shared_snapshot_when_discovering_project_then_matches_isolated_discovery( + tmp_path: Path, test_case: SharedSnapshotDiscoveryTestCase +) -> None: + write_scoped_project(root=tmp_path) + + mismatches: tuple[str, ...] = isolated_discovery_mismatches(project_dir=tmp_path) + + assert mismatches == test_case.expected_mismatches + assert model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) == ( + test_case.expected_model_paths + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoveryPassRefreshTestCase( + "added_and_removed_models", + added_files={ + "models/sales/weekly.sql": "MODEL (description 'Weekly.');\nSELECT 1 AS week_id\n" + }, + removed_path="models/orders.sql", + expected_first_paths=("models/orders.sql", "models/sales/daily.sql"), + expected_second_paths=("models/sales/daily.sql", "models/sales/weekly.sql"), + ) + ], + ids=lambda case: case.description, +) +def test_given_files_change_between_discovery_passes_when_discovering_then_sees_current_tree( + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], + test_case: DiscoveryPassRefreshTestCase, +) -> None: + write_scoped_project(root=tmp_path) + first: tuple[str, ...] = model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) + write_repo_files(tmp_path, test_case.added_files) + (tmp_path / test_case.removed_path).unlink() + + second: tuple[str, ...] = model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) + + assert first == test_case.expected_first_paths + assert second == test_case.expected_second_paths + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScopeReuseTestCase( + "nested_reuses_later_refreshes", + expected_nested_reuse=True, + expected_later_reuse=False, + ) + ], + ids=lambda case: case.description, +) +def test_given_nested_scope_for_same_project_when_entering_then_reuses_active_snapshot( + tmp_path: Path, test_case: SnapshotScopeReuseTestCase +) -> None: + with DirectorySnapshot.scope(project_dir=tmp_path) as outer: + with DirectorySnapshot.scope(project_dir=tmp_path) as inner: + nested_reuse: bool = inner is outer + with DirectorySnapshot.scope(project_dir=tmp_path) as later: + later_reuse: bool = later is outer + + assert nested_reuse is test_case.expected_nested_reuse + assert later_reuse is test_case.expected_later_reuse + + +@pytest.mark.parametrize( + "test_case", + [ + NativeHeaderMatchTestCase("seed_1", seed=1, count=2000), + NativeHeaderMatchTestCase("seed_2", seed=2, count=2000), + NativeHeaderMatchTestCase("seed_3", seed=3, count=2000), + NativeHeaderMatchTestCase("seed_4", seed=4, count=2000), + ], + ids=lambda case: case.description, +) +def test_given_randomized_model_files_when_matching_natively_then_matches_python_pattern( + test_case: NativeHeaderMatchTestCase, +) -> None: + mismatches: tuple[str, ...] = header_match_mismatches( + seed=test_case.seed, count=test_case.count + ) + + assert mismatches == test_case.expected_mismatches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 977c292d474e082365a692eaab8020355f27e973 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:23:03 +0100 Subject: [PATCH 3/4] perf(compile): compare warm artifacts in one open and prune only empty folders Warm writes compared every compiled artifact with a stat followed by a separate read; they now open each file once. The stale-file walk tried to remove every compiled directory and relied on failing rmdir calls for non-empty ones; it now removes only folders left empty by pruning, with the same result. Unchanged artifacts keep their bytes and mtimes. --- .../_helpers/compile/target_writer.py | 41 +++++-- .../cli/commands/main/_test_helpers.py | 87 +++++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 19 ++++ .../commands/main/test_compile_warm_target.py | 100 ++++++++++++++++++ 4 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index aa95dc2d60..ecfadd0214 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -5,6 +5,7 @@ import errno import json import os +import stat import uuid from collections.abc import Callable from pathlib import Path @@ -674,11 +675,12 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - if check_existing and path.is_file(): - existing: bytes = path.read_bytes() + if check_existing: + existing: bytes | None = _read_existing_file(path=path) if existing == contents: return - _ = existing.decode("utf-8") + if existing is not None: + _ = existing.decode("utf-8") try: _overwrite_bytes(path=path, contents=contents) except FileNotFoundError: @@ -686,6 +688,18 @@ def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool _overwrite_bytes(path=path, contents=contents) +def _read_existing_file(*, path: Path) -> bytes | None: + """Return a regular file's bytes in one open, or None when there is no file to compare.""" + + try: + with open(path, "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + return None + return handle.read() + except (FileNotFoundError, NotADirectoryError, IsADirectoryError): + return None + + def _overwrite_text(*, path: Path, contents: str) -> None: _ = path.write_text(contents, encoding="utf-8") @@ -712,21 +726,30 @@ def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path]) if not compiled_dir.is_dir(): return managed_names: set[str] = {os.fspath(path) for path in managed_paths} + removed_directories: set[str] = set() for root, directories, filenames in os.walk(compiled_dir, topdown=False): + kept_file: bool = False for filename in filenames: path: str = os.path.join(root, filename) - if path not in managed_names: + if path in managed_names: + kept_file = True + else: os.unlink(path) - for name in directories: - _remove_empty_directory(os.path.join(root, name)) - _remove_empty_directory(compiled_dir) + if not kept_file and all( + os.path.join(root, name) in removed_directories for name in directories + ): + if root == os.fspath(compiled_dir) or _remove_empty_directory(root): + removed_directories.add(root) + if os.fspath(compiled_dir) in removed_directories: + _remove_empty_directory(compiled_dir) -def _remove_empty_directory(directory: str | Path) -> None: +def _remove_empty_directory(directory: str | Path) -> bool: try: os.rmdir(directory) except OSError: - pass + return False + return True def _model_output_path(relative_path: Path) -> Path: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py new file mode 100644 index 0000000000..8f6eddcf43 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py @@ -0,0 +1,87 @@ +"""Warm compile target fixtures and artifact inspection for target reconciliation tests.""" + +from __future__ import annotations + +import os +from itertools import compress +from pathlib import Path + +from sqlbuild.cli.commands.main.entrypoint.entry import main + +_WARM_TARGET_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "shop"\nadapter = "duckdb"\n', + "models/sales/orders.sql": ( + "MODEL (description 'Test model orders.', columns (order_id (type INTEGER, " + "audits [not_null])));\nSELECT 1 AS order_id\n" + ), + "models/sales/daily/totals.sql": ( + "MODEL (description 'Test model totals.');\nSELECT order_id FROM __ref(\"orders\")\n" + ), + "models/inventory.sql": "MODEL (description 'Test model inventory.');\nSELECT 2 AS item_id\n", +} + + +def write_warm_target_project(*, project_dir: Path) -> None: + """Write a small project with nested models and one attached audit.""" + + for relative_path, contents in _WARM_TARGET_PROJECT_FILES.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def compile_project(*, project_dir: Path, compile_args: tuple[str, ...]) -> int: + """Run the real compile command and return its exit code.""" + + return main(["--project-dir", str(project_dir), "--no-color", "compile", *compile_args]) + + +def compiled_artifacts(*, compiled_dir: Path) -> tuple[Path, ...]: + """Return every regular file below target/compiled, without following directory links.""" + + artifacts: list[Path] = [] + for root, _directories, names in os.walk(compiled_dir): + artifacts.extend(Path(root, name) for name in names) + return tuple(sorted(artifacts)) + + +def stamp_artifacts(*, artifacts: tuple[Path, ...], mtime_ns: int) -> None: + """Give every artifact one known modification time.""" + + for path in artifacts: + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def changed_artifacts(*, compiled_dir: Path, mtime_ns: int) -> tuple[str, ...]: + """Return artifacts whose modification time moved away from the stamped value.""" + + artifacts: tuple[Path, ...] = compiled_artifacts(compiled_dir=compiled_dir) + changed: list[bool] = [path.stat().st_mtime_ns != mtime_ns for path in artifacts] + return tuple(path.relative_to(compiled_dir).as_posix() for path in compress(artifacts, changed)) + + +def pruned_entries(*, entries: dict[str, Path], names: tuple[str, ...]) -> tuple[str, ...]: + """Return the named stale entries that no longer exist.""" + + return tuple(compress(names, [not entries[name].exists() for name in names])) + + +def write_stale_target_entries(*, compiled_dir: Path, outside_dir: Path) -> dict[str, Path]: + """Add a stale nested artifact, an empty folder, and a link to an outside folder.""" + + stale_file: Path = compiled_dir / "models" / "legacy" / "archive" / "returns.sql" + stale_file.parent.mkdir(parents=True) + stale_file.write_text("SELECT 'stale'\n", encoding="utf-8") + empty_directory: Path = compiled_dir / "models" / "empty" / "nested" + empty_directory.mkdir(parents=True) + outside_dir.mkdir() + (outside_dir / "kept.sql").write_text("SELECT 'kept'\n", encoding="utf-8") + link: Path = compiled_dir / "models" / "linked" / "shared" + link.parent.mkdir(parents=True) + link.symlink_to(outside_dir, target_is_directory=True) + return { + "legacy": compiled_dir / "models" / "legacy", + "empty": compiled_dir / "models" / "empty", + "link": link, + "outside_file": outside_dir / "kept.sql", + } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index af6a8c559d..4bf59b42a7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -910,3 +910,22 @@ class UnusedOutputDifferentialCase: cte_sql: str reader_sql: str expected_statuses: tuple[str, ...] + + +@dataclass(frozen=True) +class WarmTargetTestCase: + description: str + compile_args: tuple[str, ...] + expected_exit_code: int = 0 + expected_changed: tuple[str, ...] = () + expected_pruned: tuple[str, ...] = ("legacy", "empty") + expected_link_kept: bool = True + + +@dataclass(frozen=True) +class WarmTargetEditTestCase: + description: str + edited_path: str + edited_contents: str + expected_changed: tuple[str, ...] + expected_exit_code: int = 0 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py new file mode 100644 index 0000000000..6af6a5fbb1 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py @@ -0,0 +1,100 @@ +"""End-to-end coverage for warm compile target reconciliation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_helpers import ( + changed_artifacts, + compile_project, + compiled_artifacts, + pruned_entries, + stamp_artifacts, + write_stale_target_entries, + write_warm_target_project, +) +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + WarmTargetEditTestCase, + WarmTargetTestCase, +) + +_UNCHANGED_MTIME_NS: int = 1_000_000_000 + + +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetTestCase("compile_cache_enabled", compile_args=()), + WarmTargetTestCase("compile_cache_disabled", compile_args=("--no-cache",)), + ], + ids=lambda case: case.description, +) +def test_given_warm_target_with_stale_entries_when_compiling_then_preserves_unchanged_and_prunes( + tmp_path: Path, capsys: pytest.CaptureFixture[str], test_case: WarmTargetTestCase +) -> None: + project_dir: Path = tmp_path / "project" + write_warm_target_project(project_dir=project_dir) + first_exit_code: int = compile_project( + project_dir=project_dir, compile_args=test_case.compile_args + ) + compiled_dir: Path = project_dir / "target" / "compiled" + managed: tuple[Path, ...] = compiled_artifacts(compiled_dir=compiled_dir) + stamp_artifacts(artifacts=managed, mtime_ns=_UNCHANGED_MTIME_NS) + stale: dict[str, Path] = write_stale_target_entries( + compiled_dir=compiled_dir, outside_dir=tmp_path / "shared" + ) + + second_exit_code: int = compile_project( + project_dir=project_dir, compile_args=test_case.compile_args + ) + _ = capsys.readouterr() + + assert (first_exit_code, second_exit_code) == ( + test_case.expected_exit_code, + test_case.expected_exit_code, + ) + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_changed + ) + assert compiled_artifacts(compiled_dir=compiled_dir) == managed + assert pruned_entries(entries=stale, names=("legacy", "empty")) == test_case.expected_pruned + assert stale["link"].is_symlink() is test_case.expected_link_kept + assert stale["outside_file"].is_file() is test_case.expected_link_kept + + +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetEditTestCase( + "one_model_edited", + edited_path="models/inventory.sql", + edited_contents="MODEL (description 'Test model inventory.');\nSELECT 3 AS item_id\n", + expected_changed=("models/inventory.sql",), + ) + ], + ids=lambda case: case.description, +) +def test_given_one_edited_model_when_compiling_warm_then_rewrites_only_its_artifacts( + tmp_path: Path, capsys: pytest.CaptureFixture[str], test_case: WarmTargetEditTestCase +) -> None: + write_warm_target_project(project_dir=tmp_path) + _ = compile_project(project_dir=tmp_path, compile_args=()) + compiled_dir: Path = tmp_path / "target" / "compiled" + stamp_artifacts( + artifacts=compiled_artifacts(compiled_dir=compiled_dir), mtime_ns=_UNCHANGED_MTIME_NS + ) + (tmp_path / test_case.edited_path).write_text(test_case.edited_contents, encoding="utf-8") + + exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + _ = capsys.readouterr() + + assert exit_code == test_case.expected_exit_code + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_changed + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 38cb96bb8db8bd1ec0738dd388466c020b0c2cc6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:24:06 +0100 Subject: [PATCH 4/4] docs: explain which path default describes a model The format page said a model described by [defaults] passes the description check, but [defaults] rejects description with D001. Project configuration now documents that each model uses only its nearest matching path default, that path defaults are not merged, and how to describe models when a broad "**" key coexists with more specific keys. --- .../sqlbuild/references/docs/cli/format.md | 5 ++- .../docs/concepts/project-configuration.md | 42 ++++++++++++++++++- website/src/content/docs/docs/cli/format.mdx | 5 ++- .../docs/concepts/project-configuration.mdx | 42 ++++++++++++++++++- 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 56243ac9ac..1662f04eb3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -50,7 +50,10 @@ line_width = 100 `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; `sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it before compile does (see [Descriptions](../concepts/models/configuration.md#descriptions)). A model -described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +described by its `model_schema`, or by the one `[path_defaults]` entry that applies to it, passes, +exactly as it compiles. `[defaults]` does not accept `description` (`D001`), and only the nearest +matching path default applies (see +[Which path default applies](../concepts/project-configuration.md#which-path-default-applies)). Format does not render templates, so a templated description that renders empty passes format and is reported by `sqb compile`. Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index a772fdacb5..e5cfd5e06e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -386,14 +386,52 @@ replay_on_change = "full" ``` Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the -`staging` path default. +`staging` path default. A key matches its folder and every folder below it. In a key, `*` matches one +folder name and `**` matches any number of folders, so `"**"` matches every model. + +### Which path default applies + +Each model uses at most one path default: the nearest matching key. Path defaults are not merged, so +a model gets nothing from a broader key when a more specific key also matches it. + +1. A key without wildcards beats every wildcard key. Among those, the deepest key wins, so + `staging/orders` beats `staging`. +2. When only wildcard keys match, the most specific wins: more literal folder names first, then more + `*` segments, then fewer `**` segments. Two equally specific matches fail compilation. + +A broad `"**"` default therefore applies only to models that no other key matches. With the +configuration below, `models/orders.sql` gets the `"**"` description, but +`models/staging/stg_customers.sql` uses only the `staging` entry. It has no description and fails +with `P010`: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +``` + +Descriptions belong to individual models, so put each one in the `MODEL()` header, or in the +`description` of the model's `model_schema`. `[defaults]` rejects `description` with `D001`. If a +folder of models really shares one description, every key those models can select must set it: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +description = "Staging model." +``` ### Config layering order Configuration is layered in this order, with later layers overriding earlier ones: 1. **Project defaults** (`defaults`) -2. **Path defaults** (`path_defaults`) - if the model's path matches +2. **Path defaults** (`path_defaults`) - the one nearest matching entry, if any (see + [Which path default applies](#which-path-default-applies)) 3. **MODEL() header** - the model's own config Most keys are overridden by the more specific layer, but three merge instead: diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index b9417cff64..0cef3bee65 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -42,7 +42,10 @@ line_width = 100 `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; `sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it before compile does (see [Descriptions](/docs/concepts/models/configuration#descriptions)). A model -described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +described by its `model_schema`, or by the one `[path_defaults]` entry that applies to it, passes, +exactly as it compiles. `[defaults]` does not accept `description` (`D001`), and only the nearest +matching path default applies (see +[Which path default applies](/docs/concepts/project-configuration#which-path-default-applies)). Format does not render templates, so a templated description that renders empty passes format and is reported by `sqb compile`. Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 8a888be1b1..8cffb29913 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -363,14 +363,52 @@ replay_on_change = "full" ``` Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the -`staging` path default. +`staging` path default. A key matches its folder and every folder below it. In a key, `*` matches one +folder name and `**` matches any number of folders, so `"**"` matches every model. + +### Which path default applies + +Each model uses at most one path default: the nearest matching key. Path defaults are not merged, so +a model gets nothing from a broader key when a more specific key also matches it. + +1. A key without wildcards beats every wildcard key. Among those, the deepest key wins, so + `staging/orders` beats `staging`. +2. When only wildcard keys match, the most specific wins: more literal folder names first, then more + `*` segments, then fewer `**` segments. Two equally specific matches fail compilation. + +A broad `"**"` default therefore applies only to models that no other key matches. With the +configuration below, `models/orders.sql` gets the `"**"` description, but +`models/staging/stg_customers.sql` uses only the `staging` entry. It has no description and fails +with `P010`: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +``` + +Descriptions belong to individual models, so put each one in the `MODEL()` header, or in the +`description` of the model's `model_schema`. `[defaults]` rejects `description` with `D001`. If a +folder of models really shares one description, every key those models can select must set it: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +description = "Staging model." +``` ### Config layering order Configuration is layered in this order, with later layers overriding earlier ones: 1. **Project defaults** (`defaults`) -2. **Path defaults** (`path_defaults`) - if the model's path matches +2. **Path defaults** (`path_defaults`) - the one nearest matching entry, if any (see + [Which path default applies](#which-path-default-applies)) 3. **MODEL() header** - the model's own config Most keys are overridden by the more specific layer, but three merge instead: