diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 59b33f5732..260f595971 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -376,6 +376,19 @@ fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult, + contents: Vec, +) -> PyResult>> { + py.compiler_detach(|| { + Ok(crate::compiler::main::model_header_matching::match_batch( + &contents, + )) + }) + .map_err(value_error) +} + #[pyfunction] fn tokenize_model_header(header: &str) -> PyResult> { compiler_guard(|| { @@ -492,6 +505,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?; + module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?; module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs new file mode 100644 index 0000000000..df18220cc1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs @@ -0,0 +1,145 @@ +//! Exact native equivalent of the possessive Python `MODEL(...)` header regular expression. + +/// Code-point offsets of one matched header: header start, header end, and SQL start. +pub(crate) type HeaderMatchOffsets = (usize, usize, usize); + +const MODEL_KEYWORD: &[u8] = b"MODEL"; + +pub(crate) fn match_one(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut index = skip_whitespace(text, 0); + if !bytes[index..].starts_with(MODEL_KEYWORD) { + return None; + } + index = skip_whitespace(text, index + MODEL_KEYWORD.len()); + if bytes.get(index) != Some(&b'(') { + return None; + } + let header_start = index + 1; + let header_end = scan_header_body(text, header_start); + if bytes.get(header_end) != Some(&b')') { + return None; + } + index = skip_whitespace(text, header_end + 1); + if bytes.get(index) != Some(&b';') { + return None; + } + let sql_start = skip_whitespace(text, index + 1); + Some(code_point_offsets( + text, + header_start, + header_end, + sql_start, + )) +} + +/// Consume the possessive header body and return the byte offset where it stops. +fn scan_header_body(text: &str, start: usize) -> usize { + let bytes = text.as_bytes(); + let mut index = start; + while let Some(&byte) = bytes.get(index) { + index = match byte { + b'"' | b'\'' => quoted_string_end(text, index, byte) + .unwrap_or_else(|| unclosed_quote_end(text, index)), + b')' => { + let after = skip_whitespace(text, index + 1); + if bytes.get(after) == Some(&b';') { + return index; + } + index + 1 + } + b'\\' => match escape_end(text, index) { + Some(end) => end, + None => return index, + }, + _ => index + 1, + }; + } + index +} + +/// `"(?:[^"\\]|\\.)*+"`: the closing quote's end, or `None` when the string never closes. +fn quoted_string_end(text: &str, open: usize, quote: u8) -> Option { + let bytes = text.as_bytes(); + let mut index = open + 1; + while let Some(&byte) = bytes.get(index) { + if byte == quote { + return Some(index + 1); + } + if byte == b'\\' { + index = escape_end(text, index)?; + } else { + index += 1; + } + } + None +} + +/// `["'](?:[^\\)]|\\.)*+`: an unclosed quote runs to the next `)` or unescapable backslash. +fn unclosed_quote_end(text: &str, open: usize) -> usize { + let bytes = text.as_bytes(); + let mut index = open + 1; + while let Some(&byte) = bytes.get(index) { + match byte { + b')' => return index, + b'\\' => match escape_end(text, index) { + Some(end) => index = end, + None => return index, + }, + _ => index += 1, + } + } + index +} + +/// `\\.` with DOTALL: a backslash plus one following code point, if any. +fn escape_end(text: &str, backslash: usize) -> Option { + text[backslash + 1..] + .chars() + .next() + .map(|escaped| backslash + 1 + escaped.len_utf8()) +} + +fn skip_whitespace(text: &str, start: usize) -> usize { + let mut index = start; + for character in text[start..].chars() { + if !is_python_whitespace(character) { + break; + } + index += character.len_utf8(); + } + index +} + +/// Python's `str.isspace()`, which `\s` uses for `str` patterns. +fn is_python_whitespace(character: char) -> bool { + matches!( + character, + '\t'..='\r' + | '\u{1c}'..='\u{20}' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} + +fn code_point_offsets( + text: &str, + header_start: usize, + header_end: usize, + sql_start: usize, +) -> HeaderMatchOffsets { + if text.is_ascii() { + return (header_start, header_end, sql_start); + } + let header_start_chars = text[..header_start].chars().count(); + let header_end_chars = header_start_chars + text[header_start..header_end].chars().count(); + let sql_start_chars = header_end_chars + text[header_end..sql_start].chars().count(); + (header_start_chars, header_end_chars, sql_start_chars) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs index 95639bca96..2c80363b66 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod matching; pub(crate) mod tokenization; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs index 761d058af1..4059a51c27 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod model_header_matching; pub(crate) mod model_header_parsing; pub(crate) mod model_header_tokenizing; pub(crate) mod sql_interpolation; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs b/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs new file mode 100644 index 0000000000..bbe71611bd --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs @@ -0,0 +1,8 @@ +use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; + +pub(crate) fn match_batch(contents: &[String]) -> Vec> { + contents + .iter() + .map(|text| matching::match_one(text)) + .collect() +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests.rs b/crates/sqlbuild-rules-native/src/compiler/tests.rs index 1a3a7a928d..85fc3262d1 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_header_matching.rs"] +mod model_header_matching; #[path = "tests/test_model_header_tokenization.rs"] mod model_header_tokenization; #[path = "tests/test_sql_interpolation.rs"] diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs new file mode 100644 index 0000000000..0b99b03c5e --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs @@ -0,0 +1,72 @@ +use crate::compiler::main::model_header_matching::match_batch; +use crate::compiler::tests::test_types::ModelHeaderMatchTestCase; + +#[test] +fn given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern() { + let test_cases = [ + ModelHeaderMatchTestCase { + description: "simple header", + contents: "MODEL (name orders);\nSELECT 1", + expected_offsets: Some((7, 18, 21)), + }, + ModelHeaderMatchTestCase { + description: "leading whitespace and spaced terminator", + contents: " \n MODEL\t( a ) \n ; \n SELECT 1", + expected_offsets: Some((10, 13, 21)), + }, + ModelHeaderMatchTestCase { + description: "closing parenthesis not followed by a semicolon stays in the header", + contents: "MODEL (columns (id INT));SELECT 1", + expected_offsets: Some((7, 23, 25)), + }, + ModelHeaderMatchTestCase { + description: "quoted closing parenthesis and semicolon stay in the header", + contents: "MODEL (description 'a);b');SELECT 1", + expected_offsets: Some((7, 25, 27)), + }, + ModelHeaderMatchTestCase { + description: "unclosed quote runs to the next parenthesis", + contents: "MODEL (description 'open);SELECT 1", + expected_offsets: Some((7, 24, 26)), + }, + ModelHeaderMatchTestCase { + description: "escaped quote inside a string", + contents: "MODEL (description \"a\\\"b\");SELECT 1", + expected_offsets: Some((7, 25, 27)), + }, + ModelHeaderMatchTestCase { + description: "non ascii offsets are code points", + contents: "MODEL (description 'caf\u{e9}')\u{a0};\u{3000}SELECT 1", + expected_offsets: Some((7, 25, 29)), + }, + ModelHeaderMatchTestCase { + description: "missing terminator", + contents: "MODEL (name orders)\nSELECT 1", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "trailing lone backslash", + contents: "MODEL (name orders\\", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "lowercase keyword", + contents: "model (name orders);", + expected_offsets: None, + }, + ModelHeaderMatchTestCase { + description: "empty file", + contents: "", + expected_offsets: None, + }, + ]; + + for test_case in test_cases { + let actual = match_batch(&[test_case.contents.to_string()])[0]; + assert_eq!( + actual, test_case.expected_offsets, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs index dff917362b..bf566c43ff 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs @@ -86,3 +86,9 @@ pub(crate) struct RelationMarkerOracleTestCase { pub(crate) dialect: &'static str, pub(crate) expected_mismatches: &'static [&'static str], } + +pub(crate) struct ModelHeaderMatchTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) expected_offsets: Option<(usize, usize, usize)>, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 56243ac9ac..1662f04eb3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -50,7 +50,10 @@ line_width = 100 `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; `sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it before compile does (see [Descriptions](../concepts/models/configuration.md#descriptions)). A model -described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +described by its `model_schema`, or by the one `[path_defaults]` entry that applies to it, passes, +exactly as it compiles. `[defaults]` does not accept `description` (`D001`), and only the nearest +matching path default applies (see +[Which path default applies](../concepts/project-configuration.md#which-path-default-applies)). Format does not render templates, so a templated description that renders empty passes format and is reported by `sqb compile`. Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index a772fdacb5..e5cfd5e06e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -386,14 +386,52 @@ replay_on_change = "full" ``` Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the -`staging` path default. +`staging` path default. A key matches its folder and every folder below it. In a key, `*` matches one +folder name and `**` matches any number of folders, so `"**"` matches every model. + +### Which path default applies + +Each model uses at most one path default: the nearest matching key. Path defaults are not merged, so +a model gets nothing from a broader key when a more specific key also matches it. + +1. A key without wildcards beats every wildcard key. Among those, the deepest key wins, so + `staging/orders` beats `staging`. +2. When only wildcard keys match, the most specific wins: more literal folder names first, then more + `*` segments, then fewer `**` segments. Two equally specific matches fail compilation. + +A broad `"**"` default therefore applies only to models that no other key matches. With the +configuration below, `models/orders.sql` gets the `"**"` description, but +`models/staging/stg_customers.sql` uses only the `staging` entry. It has no description and fails +with `P010`: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +``` + +Descriptions belong to individual models, so put each one in the `MODEL()` header, or in the +`description` of the model's `model_schema`. `[defaults]` rejects `description` with `D001`. If a +folder of models really shares one description, every key those models can select must set it: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +description = "Staging model." +``` ### Config layering order Configuration is layered in this order, with later layers overriding earlier ones: 1. **Project defaults** (`defaults`) -2. **Path defaults** (`path_defaults`) - if the model's path matches +2. **Path defaults** (`path_defaults`) - the one nearest matching entry, if any (see + [Which path default applies](#which-path-default-applies)) 3. **MODEL() header** - the model's own config Most keys are overridden by the more specific layer, but three merge instead: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d90c492dac..e132ef6d4d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -31,6 +31,7 @@ def load_config_json(project_dir: str) -> str: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... def skill_freshness(content: str | None, input_fingerprint: str) -> str: ... +def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index aa95dc2d60..ecfadd0214 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -5,6 +5,7 @@ import errno import json import os +import stat import uuid from collections.abc import Callable from pathlib import Path @@ -674,11 +675,12 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - if check_existing and path.is_file(): - existing: bytes = path.read_bytes() + if check_existing: + existing: bytes | None = _read_existing_file(path=path) if existing == contents: return - _ = existing.decode("utf-8") + if existing is not None: + _ = existing.decode("utf-8") try: _overwrite_bytes(path=path, contents=contents) except FileNotFoundError: @@ -686,6 +688,18 @@ def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool _overwrite_bytes(path=path, contents=contents) +def _read_existing_file(*, path: Path) -> bytes | None: + """Return a regular file's bytes in one open, or None when there is no file to compare.""" + + try: + with open(path, "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + return None + return handle.read() + except (FileNotFoundError, NotADirectoryError, IsADirectoryError): + return None + + def _overwrite_text(*, path: Path, contents: str) -> None: _ = path.write_text(contents, encoding="utf-8") @@ -712,21 +726,30 @@ def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path]) if not compiled_dir.is_dir(): return managed_names: set[str] = {os.fspath(path) for path in managed_paths} + removed_directories: set[str] = set() for root, directories, filenames in os.walk(compiled_dir, topdown=False): + kept_file: bool = False for filename in filenames: path: str = os.path.join(root, filename) - if path not in managed_names: + if path in managed_names: + kept_file = True + else: os.unlink(path) - for name in directories: - _remove_empty_directory(os.path.join(root, name)) - _remove_empty_directory(compiled_dir) + if not kept_file and all( + os.path.join(root, name) in removed_directories for name in directories + ): + if root == os.fspath(compiled_dir) or _remove_empty_directory(root): + removed_directories.add(root) + if os.fspath(compiled_dir) in removed_directories: + _remove_empty_directory(compiled_dir) -def _remove_empty_directory(directory: str | Path) -> None: +def _remove_empty_directory(directory: str | Path) -> bool: try: os.rmdir(directory) except OSError: - pass + return False + return True def _model_output_path(relative_path: Path) -> Path: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 1593bb20b8..b1f921b905 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -29,20 +29,22 @@ from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( resolve_scope_path_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, GrantRecord, ResourceIdentity, ResourceRecord, ScopeIndex, ScopeLookup, - VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -288,7 +290,9 @@ def _shared_visible_declarations( def _resolve_shared_declarations( *, lookup: ScopeLookup, resource: ResourceIdentity ) -> tuple[DeclarationRecord, ...]: - resolution: VisibilityResolution = resolve_scope_visibility(lookup=lookup, target=resource) + resolution: DeclarationVisibility = resolve_scope_declaration_visibility( + lookup=lookup, target=resource + ) declarations: list[DeclarationRecord] = [] for visible in resolution.visible: records: tuple[DeclarationRecord, ...] = lookup.declarations.get(visible.declaration, ()) @@ -314,7 +318,7 @@ def _tested_macro_grants( if actual_cte is None: return [] tested_macro_names: tuple[str, ...] = find_macro_call_names(actual_cte.sql_body) - direct_resolution: VisibilityResolution = resolve_scope_visibility( + direct_resolution: DeclarationVisibility = resolve_scope_declaration_visibility( lookup=lookup, target=resource ) directly_visible: frozenset[DeclarationIdentity] = frozenset( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index e6c27ec574..c9678e7cb9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -34,20 +34,22 @@ from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( resolve_scope_path_visibility, ) -from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, ResourceIdentity, ResourceRecord, ScopeIndex, UsageRecord, VisibilityRecord, - VisibilityResolution, ) from sqlbuild.compiler.scopes.types import ( DeclarationKind, @@ -197,7 +199,7 @@ def resolve_declaration_context( ) if cached_path_context is not None: return cached_path_context - resolution: VisibilityResolution = resolve_scope_visibility( + resolution: DeclarationVisibility = resolve_scope_declaration_visibility( lookup=resolver.lookup, target=resource or target_path ) enums: dict[str, EnumDeclaration] = {} @@ -218,7 +220,7 @@ def resolve_declaration_context( resolver.lookup.declarations[item.declaration][0] for item in resolution.visible ) inaccessible_records: tuple[DeclarationRecord, ...] = tuple( - resolver.lookup.declarations[item.declaration][0] for item in resolution.inaccessible + resolver.lookup.declarations[identity][0] for identity in resolution.inaccessible ) for visible_record in resolution.visible: visibility_by_declaration.setdefault(visible_record.declaration, []).append( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index d8026940b2..d2e37bec65 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -34,6 +34,7 @@ build_integration_loader_functions, ) from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import ( DiscoveredAdapterFile, DiscoveredAssetFunction, @@ -81,9 +82,12 @@ def build_discovered_project_inputs( ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" - with OperationLifecycle( - operation_kind="project", operation_name="discovery_declaration_parse" - ) as declaration_lifecycle: + with ( + OperationLifecycle( + operation_kind="project", operation_name="discovery_declaration_parse" + ) as declaration_lifecycle, + DirectorySnapshot.scope(project_dir=project_dir), + ): source_files: tuple[DiscoveredSourceFile, ...] = discover_source_files( project_dir=project_dir, fact_cache=fact_cache ) @@ -227,6 +231,11 @@ def build_discovered_project_inputs( def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" + with DirectorySnapshot.scope(project_dir=project_dir): + return _build_tolerant_scope_discovery(project_dir=project_dir) + + +def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: project_config, local_config, config_faults = _discover_configs(project_dir=project_dir) models, model_faults = _discover_models(project_dir=project_dir) enums, constants, macros, declaration_faults = _discover_declarations(project_dir=project_dir) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 0abb5c741f..30f325738c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -5,14 +5,13 @@ import ast import importlib.util import inspect -import re import sys from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass, field from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import get_type_hints +from typing import cast, get_type_hints from pydantic import ValidationError @@ -26,6 +25,10 @@ named_declaration_files, named_declaration_roots, ) +from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( + is_in_scoped_declaration_tree, + project_relative_path, +) from sqlbuild.compiler.discovery._helpers.filesystem.sql_test_files import discover_sql_test_files from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file @@ -37,18 +40,18 @@ from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - match_model_header, + match_model_headers, prepare_matched_model_file_headers, ) from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SCHEMA_FILE_NAME, SEED_FILE_SUFFIX, - SQL_TESTS_OWNERSHIP_ROOT, YAML_FILE_SUFFIXES, ) from sqlbuild.compiler.discovery.exceptions import ( @@ -89,6 +92,7 @@ DiscoveredSqlTestFile, DiscoveredTaskFunction, DiscoveryFileFault, + ModelHeaderMatch, NamedDeclarationRoot, ) from sqlbuild.compiler.discovery.types import ScopedDeclarationFile @@ -101,8 +105,8 @@ DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, GROUPED_NAMED_DECLARATION_DIRECTORIES, - INHERITED_DECLARATION_DIRECTORIES, LOCAL_DECLARATION_DIRECTORIES, + SCOPED_DECLARATION_DIRECTORIES, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.provider.classes.provider import Provider @@ -135,6 +139,7 @@ from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" +_FACTS_MEMO_KEY: str = "declaration_file_facts" @dataclass @@ -172,18 +177,25 @@ class _DeclarationFileFacts: declaration_root: Path -_SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( - INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES -) -_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" -_SQL_FILE_SUFFIX: str = ".sql" -_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts - - def _discover_declaration_file_facts( *, project_dir: Path, declaration_kind: DeclarationKind | None = None ) -> tuple[_DeclarationFileFacts, ...]: - _validate_declaration_groups(project_dir=project_dir) + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + memo_key: tuple[str, DeclarationKind | None] = (_FACTS_MEMO_KEY, declaration_kind) + cached: object = tree.memo.get(memo_key) + if cached is not None: + return cast(tuple[_DeclarationFileFacts, ...], cached) + facts: tuple[_DeclarationFileFacts, ...] = _scan_declaration_file_facts( + project_dir=project_dir, declaration_kind=declaration_kind, tree=tree + ) + tree.memo[memo_key] = facts + return facts + + +def _scan_declaration_file_facts( + *, project_dir: Path, declaration_kind: DeclarationKind | None, tree: DirectorySnapshot +) -> tuple[_DeclarationFileFacts, ...]: + _validate_declaration_groups(project_dir=project_dir, tree=tree) for directory_name in sorted(LOCAL_DECLARATION_DIRECTORIES): directory_kind, _scope_kind = DECLARATION_DIRECTORY_FACTS[directory_name] if (declaration_kind is None or directory_kind is declaration_kind) and ( @@ -204,6 +216,7 @@ def _discover_declaration_file_facts( facts.extend( _declaration_files_under_root( project_dir=project_dir, + tree=tree, ownership_root=Path(global_directory), declaration_root=declaration_root, owning_path=None, @@ -217,17 +230,17 @@ def _discover_declaration_file_facts( if not authored_root.is_dir(): continue directory: Path - for directory in sorted(path for path in authored_root.rglob("*") if path.is_dir()): - if directory.name not in _SCOPED_DECLARATION_DIRECTORIES: + for directory in sorted(tree.directories(root=authored_root)): + if directory.name not in SCOPED_DECLARATION_DIRECTORIES: continue directory_kind, directory_scope_kind = DECLARATION_DIRECTORY_FACTS[directory.name] if declaration_kind is not None and directory_kind is not declaration_kind: continue - relative_directory: Path = _project_relative_path( + relative_directory: Path = project_relative_path( path=directory, project_dir=project_dir ) descendants: tuple[str, ...] = relative_directory.parts[len(root_components) :] - if any(part in _SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): + if any(part in SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): raise DeclarationParseError( f"Declaration root {relative_directory.as_posix()}/ is nested inside another " "declaration tree" @@ -238,6 +251,7 @@ def _discover_declaration_file_facts( facts.extend( _declaration_files_under_root( project_dir=project_dir, + tree=tree, ownership_root=Path(*root_components), declaration_root=directory, owning_path=owning_path, @@ -247,7 +261,7 @@ def _discover_declaration_file_facts( return tuple(sorted(facts, key=lambda item: item.relative_path.as_posix())) -def _validate_declaration_groups(*, project_dir: Path) -> None: +def _validate_declaration_groups(*, project_dir: Path, tree: DirectorySnapshot) -> None: root_group: Path = project_dir / DECLARATION_GROUP_DIRECTORY if root_group.exists(): raise DeclarationParseError( @@ -259,7 +273,9 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: if not authored_root.is_dir(): continue for group in sorted( - path for path in authored_root.rglob(DECLARATION_GROUP_DIRECTORY) if path.is_dir() + path + for path in tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY) + if path.is_dir() ): if group.parent == authored_root: raise DeclarationParseError( @@ -273,12 +289,12 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: for child in group.iterdir() if not child.is_dir() or child.name - not in _SCOPED_DECLARATION_DIRECTORIES | GROUPED_NAMED_DECLARATION_DIRECTORIES + not in SCOPED_DECLARATION_DIRECTORIES | GROUPED_NAMED_DECLARATION_DIRECTORIES ) ) if unsupported: rendered: str = ", ".join( - _project_relative_path(path=path, project_dir=project_dir).as_posix() + project_relative_path(path=path, project_dir=project_dir).as_posix() for path in unsupported ) raise DeclarationParseError( @@ -290,6 +306,7 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: def _declaration_files_under_root( *, project_dir: Path, + tree: DirectorySnapshot, ownership_root: Path, declaration_root: Path, owning_path: Path | None, @@ -298,10 +315,10 @@ def _declaration_files_under_root( nested_root: Path for nested_root in sorted( path - for path in declaration_root.rglob("*") - if path.is_dir() and path.name in _SCOPED_DECLARATION_DIRECTORIES + for path in tree.directories(root=declaration_root) + if path.name in SCOPED_DECLARATION_DIRECTORIES ): - relative_nested_root: str = _project_relative_path( + relative_nested_root: str = project_relative_path( path=nested_root, project_dir=project_dir ).as_posix() raise DeclarationParseError( @@ -311,18 +328,18 @@ def _declaration_files_under_root( suffix: str = ".py" if declaration_kind is DeclarationKind.MACRO else ".sql" results: list[_DeclarationFileFacts] = [] file_path: Path - for file_path in sorted(declaration_root.rglob(f"*{suffix}")): + for file_path in sorted(tree.rglob(root=declaration_root, pattern=f"*{suffix}")): if declaration_kind is DeclarationKind.MACRO and file_path.stem == PYTHON_INIT_MODULE_STEM: continue results.append( _DeclarationFileFacts( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), declaration_kind=declaration_kind, scope_kind=scope_kind, ownership_root=ownership_root, owning_path=owning_path, - declaration_root=_project_relative_path( + declaration_root=project_relative_path( path=declaration_root, project_dir=project_dir ), ) @@ -330,28 +347,6 @@ def _declaration_files_under_root( return results -def _is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: - relative_parts: tuple[str, ...] = _project_relative_path( - path=file_path, project_dir=project_dir - ).parts - root_components: tuple[str, ...] - for root_components in CANONICAL_AUTHORED_ROOTS: - if relative_parts[: len(root_components)] != root_components: - continue - scoped_components: tuple[str, ...] = relative_parts[len(root_components) : -1] - if ( - root_components == _SQL_TEST_ROOT_COMPONENTS - and file_path.suffix == _SQL_FILE_SUFFIX - and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) - ): - return False - return any( - component == DECLARATION_GROUP_DIRECTORY or component in _SCOPED_DECLARATION_DIRECTORIES - for component in scoped_components - ) - return False - - def discover_model_files( *, project_dir: Path, @@ -368,25 +363,33 @@ def discover_model_files( loaded_model_files: list[tuple[Path, str | None, Exception | None]] = [] file_path: Path - for file_path in sorted(model_root.rglob("*.sql")): + for file_path in _sorted_glob(project_dir=project_dir, root=model_root, pattern="*.sql"): if selected_model_names is not None and file_path.stem not in selected_model_names: continue - if _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): + if is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): continue try: loaded_model_files.append((file_path, file_path.read_text(encoding="utf-8"), None)) except (OSError, UnicodeError, ValueError, SyntaxError) as error: loaded_model_files.append((file_path, None, error)) - header_matches: list[re.Match[str] | None] = [ - match_model_header(contents) if contents is not None and error is None else None + readable_contents: list[str] = [ + contents + for _path, contents, error in loaded_model_files + if contents is not None and error is None + ] + readable_matches: Iterator[ModelHeaderMatch | None] = iter( + match_model_headers(readable_contents) + ) + header_matches: list[ModelHeaderMatch | None] = [ + next(readable_matches) if contents is not None and error is None else None for _path, contents, error in loaded_model_files ] prepare_matched_model_file_headers(header_matches) discovered_model_files: list[DiscoveredSqlModelFile] = [] contents: str | None read_error: Exception | None - header_match: re.Match[str] | None + header_match: ModelHeaderMatch | None for (file_path, contents, read_error), header_match in zip( loaded_model_files, header_matches, strict=True ): @@ -401,7 +404,7 @@ def discover_model_files( discovered_model_files.append( discover_matched_model_file( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_match=header_match, extract_implicit_alias_columns=extract_implicit_alias_columns, @@ -535,8 +538,8 @@ def _parse_discovered_paths[DiscoveredT]( def _unscoped_files(*, root: Path, pattern: str, project_dir: Path) -> Iterator[Path]: file_path: Path - for file_path in sorted(root.rglob(pattern)): - if not _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): + for file_path in _sorted_glob(project_dir=project_dir, root=root, pattern=pattern): + if not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): yield file_path @@ -548,7 +551,7 @@ def discover_model_schema_files( def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredModelSchemaFile: file_path: Path = item[1] contents: str = file_path.read_text(encoding="utf-8") - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) return DiscoveredModelSchemaFile( file_path=file_path, relative_path=relative_path, @@ -581,6 +584,7 @@ def _parse_named_declaration_files[DiscoveredT: ScopedDeclarationFile]( items=( item for item in named_declaration_files( + project_dir=project_dir, roots=named_declaration_roots(project_dir=project_dir, kinds=kinds), pattern="*.sql", ) @@ -608,7 +612,7 @@ def parse(file_path: Path) -> DiscoveredSqlFunctionFile: header_values, body_sql = parse_function_sql(contents=contents, file_path=file_path) return DiscoveredSqlFunctionFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, body_sql=body_sql, @@ -633,7 +637,7 @@ def discover_python_function_files( discovered_function_files: list[DiscoveredPythonFunctionFile] = [] file_path: Path - for file_path in sorted(function_root.rglob("*.py")): + for file_path in _sorted_glob(project_dir=project_dir, root=function_root, pattern="*.py"): contents: str = file_path.read_text(encoding="utf-8") header_values: dict[str, object] entry_point: str @@ -644,7 +648,7 @@ def discover_python_function_files( discovered_function_files.append( DiscoveredPythonFunctionFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, entry_point=entry_point, @@ -657,6 +661,7 @@ def discover_python_function_files( def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: """Discover model schema.yml files and seed declaration .yml files.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) schema_paths: list[Path] = [] models_root: Path = project_dir / "models" seeds_root: Path = project_dir / "seeds" @@ -664,17 +669,17 @@ def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, . if models_root.is_dir(): schema_paths.extend( path - for path in sorted(models_root.rglob(SCHEMA_FILE_NAME)) - if not _is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) + for path in sorted(tree.rglob(root=models_root, pattern=SCHEMA_FILE_NAME)) + if not is_in_scoped_declaration_tree(file_path=path, project_dir=project_dir) ) if seeds_root.is_dir(): yaml_path: Path - for yaml_path in sorted(seeds_root.rglob("*.yaml")): + for yaml_path in sorted(tree.rglob(root=seeds_root, pattern="*.yaml")): raise SchemaParseError( f"Seed declaration file {yaml_path.relative_to(project_dir)} must use .yml; " ".yaml is not supported" ) - schema_paths.extend(sorted(seeds_root.rglob("*.yml"))) + schema_paths.extend(sorted(tree.rglob(root=seeds_root, pattern="*.yml"))) deduped_paths: tuple[Path, ...] = tuple(dict.fromkeys(schema_paths)) discovered_schema_files: list[DiscoveredSchemaFile] = [] @@ -687,7 +692,7 @@ def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, . discovered_schema_files.append( DiscoveredSchemaFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, model_entries=model_entries, seed_entries=seed_entries, @@ -716,7 +721,7 @@ def parse(file_path: Path) -> DiscoveredSourceFile: return parse_source_file_with_cache( project_dir=project_dir, file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), fact_cache=fact_cache, ) @@ -735,9 +740,9 @@ def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: return tuple( DiscoveredSeedFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) - for file_path in sorted(seeds_root.rglob("*")) + for file_path in _sorted_glob(project_dir=project_dir, root=seeds_root, pattern="*") if file_path.is_file() and file_path.suffix == SEED_FILE_SUFFIX ) @@ -757,9 +762,9 @@ def discover_test_files( file_paths: tuple[Path, ...] = tuple( file_path - for file_path in sorted(tests_root.rglob("*.sql")) + for file_path in _sorted_glob(project_dir=project_dir, root=tests_root, pattern="*.sql") if (selected_paths is None or file_path.resolve() in selected_paths) - and not _is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir) + and not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir) ) return discover_sql_test_files( project_dir=project_dir, @@ -782,7 +787,7 @@ def parse(file_path: Path) -> DiscoveredSqlScenarioFile: return parse_sql_scenario_file( contents=file_path.read_text(encoding="utf-8"), file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) return _parse_discovered_paths( @@ -803,7 +808,7 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredAuditFile: contents: str = file_path.read_text(encoding="utf-8") return DiscoveredAuditFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), contents=contents, blocks=parse_sql_audit_file(contents=contents, file_path=file_path), declaration_kind=root.kind, @@ -842,7 +847,7 @@ def discover_macro_files( def _discovery_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault: try: - relative_path: Path | None = _project_relative_path(path=path, project_dir=project_dir) + relative_path: Path | None = project_relative_path(path=path, project_dir=project_dir) except ValueError: relative_path = None message: str = str(error).replace(str(project_dir), ".") @@ -871,7 +876,7 @@ def discover_materialization_files( discovered_files.append( DiscoveredMaterializationFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=file_path.stem, provider_usages=( _provider_usages(function=materialize_fn, provider_by_name=provider_by_name) @@ -912,6 +917,7 @@ def discover_hook_functions( root: NamedDeclarationRoot file_path: Path for root, file_path in named_declaration_files( + project_dir=project_dir, roots=named_declaration_roots( project_dir=project_dir, kinds=frozenset({DeclarationKind.PYTHON_HOOK}) ), @@ -941,7 +947,7 @@ def discover_hook_functions( root.place( DiscoveredHookFunction( file_path=file_path, - relative_path=_project_relative_path( + relative_path=project_relative_path( path=file_path, project_dir=project_dir ), name=hook_definition.name, @@ -968,7 +974,7 @@ def parse(item: tuple[NamedDeclarationRoot, Path]) -> DiscoveredSqlHookFile: return parse_sql_hook_file( contents=file_path.read_text(encoding="utf-8"), file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) return _parse_named_declaration_files( @@ -1028,7 +1034,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, discovered_providers.append( DiscoveredProvider( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=provider_name, provider_class=provider_class, settings=_provider_instance( @@ -1117,7 +1123,7 @@ def discover_event_exporter_declarations( discovered.append( DiscoveredEventExporterDeclaration( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=definition.name, function=value, event_kinds=definition.event_kinds, @@ -1166,7 +1172,7 @@ def _validate_event_exporter_declaration_signature( file_path: Path, project_dir: Path, ) -> None: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if inspect.iscoroutinefunction(function): raise EventExporterDiscoveryError( f"Event exporter '{exporter_name}' in {relative_path} must be synchronous" @@ -1222,7 +1228,7 @@ def _bind_event_exporter_provider_usages( project_dir: Path, provider_by_name: dict[str, DiscoveredProvider], ) -> tuple[DiscoveredProviderUsage, ...]: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) parameters: tuple[inspect.Parameter, ...] = tuple( inspect.signature(function).parameters.values() ) @@ -1257,7 +1263,7 @@ def _public_python_files(*, root: Path) -> tuple[Path, ...]: def _provider_name(*, provider_class: type[Provider], file_path: Path, project_dir: Path) -> str: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) explicit_name: str | None = provider_class.provider_name if explicit_name is not None: validate_resource_identity( @@ -1286,7 +1292,7 @@ def _provider_instance( try: return provider_class() except ValidationError as error: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) raise ProviderDiscoveryError( f"Provider '{provider_name}' in {relative_path} has invalid settings:\n" f"{_format_provider_validation_error(error)}" @@ -1458,7 +1464,7 @@ def _restore_python_root_modules(*, saved_modules: dict[str, ModuleType]) -> Non def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: """Import a python/ module under its package name, reusing it within a discovery pass.""" - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) module_name: str = ".".join(relative_path.with_suffix("").parts) old_path: list[str] = list(sys.path) sys.path.insert(0, str(project_dir)) @@ -1556,7 +1562,7 @@ def _append_module_audit_factories( definition: AuditFactoryDefinition | None = read_audit_factory_definition(value) if definition is None: continue - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if _python_node_definition_names(value): kinds: str = ", ".join(_python_node_definition_names(value)) raise PythonNodeDiscoveryError( @@ -1607,7 +1613,7 @@ def _call_audit_factory( file_path: Path, project_dir: Path, ) -> tuple[AuditCase, ...]: - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) if inspect.signature(factory).parameters: raise PythonNodeDiscoveryError( f"Audit factory '{definition.name}' in {relative_path} must not require arguments" @@ -1648,7 +1654,7 @@ def _append_python_node_function( bucket.add_loader( DiscoveredLoaderFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=loader_definition.name, function=function, depends_on=loader_definition.depends_on, @@ -1671,7 +1677,7 @@ def _append_python_node_function( bucket.add_task( DiscoveredTaskFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=task_definition.name, function=function, depends_on=task_definition.depends_on, @@ -1692,7 +1698,7 @@ def _append_python_node_function( bucket.add_asset( DiscoveredAssetFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=asset_definition.name, function=function, depends_on=asset_definition.depends_on, @@ -1715,7 +1721,7 @@ def _append_python_node_function( bucket.add_check( DiscoveredCheckFunction( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), name=check_definition.name, function=function, depends_on=check_definition.depends_on, @@ -1795,7 +1801,7 @@ def _load_python_node_module(*, file_path: Path, project_dir: Path) -> ModuleTyp return _exec_project_module( module_name="sqlbuild_project_python_node_" + "_".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ), file_path=file_path, project_dir=project_dir, @@ -1830,7 +1836,7 @@ def _load_project_package_module( error_type: type[Exception], ) -> ModuleType: module_name: str = ".".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ) _evict_stale_project_package_modules( root_module=module_name.split(".", maxsplit=1)[0], @@ -1921,7 +1927,7 @@ def _load_materialization_module(*, file_path: Path, project_dir: Path) -> Modul raise PythonNodeDiscoveryError(removed_hook_message) module: ModuleType = _exec_project_module( module_name=".".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ), file_path=file_path, project_dir=project_dir, @@ -1942,21 +1948,16 @@ def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: return DiscoveredAdapterFile( file_path=file_path, - relative_path=_project_relative_path(path=file_path, project_dir=project_dir), + relative_path=project_relative_path(path=file_path, project_dir=project_dir), ) -def _project_relative_path(*, path: Path, project_dir: Path) -> Path: - """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" +def _sorted_glob(*, project_dir: Path, root: Path, pattern: str) -> list[Path]: + """Return ``sorted(root.rglob(pattern))`` from the active discovery snapshot.""" - root_parts: tuple[str, ...] = project_dir.parts - path_parts: tuple[str, ...] = path.parts - if ( - path.is_absolute() is not project_dir.is_absolute() - or path_parts[: len(root_parts)] != root_parts - ): - return path.relative_to(project_dir) - return path.with_segments(*path_parts[len(root_parts) :]) + return sorted( + DirectorySnapshot.current(project_dir=project_dir).rglob(root=root, pattern=pattern) + ) def _is_relative_to(*, path: Path, parent: Path) -> bool: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py index 701ce4b7e7..7a31694a5c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from pathlib import Path from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -14,7 +13,7 @@ matched_model_output_column_locations, parse_matched_model_sql, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, ModelHeaderMatch def discover_matched_model_file( @@ -22,7 +21,7 @@ def discover_matched_model_file( file_path: Path, relative_path: Path, contents: str, - header_match: re.Match[str] | None, + header_match: ModelHeaderMatch | None, extract_implicit_alias_columns: bool, extract_output_column_locations: bool, ) -> DiscoveredSqlModelFile: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index 82ed8d592a..9d0156d070 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -4,7 +4,9 @@ from collections.abc import Iterator from pathlib import Path +from typing import cast +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import CANONICAL_AUTHORED_ROOTS from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import NamedDeclarationRoot @@ -21,6 +23,8 @@ _AUDIT_ROLE_DIRECTORY: str = "audits" _LOCAL_AUDIT_ROLE_DIRECTORY: str = "_audits" _SINGULAR_AUDIT_DIRECTORY: str = "singular" +_LAYOUT_MEMO_KEY: str = "named_declaration_layout" +_GROUPS_MEMO_KEY: str = "declaration_groups" _NESTED_ROLE_DIRECTORIES: frozenset[str] = frozenset( {DECLARATION_GROUP_DIRECTORY} | INHERITED_DECLARATION_DIRECTORIES @@ -33,7 +37,10 @@ def named_declaration_roots( ) -> tuple[NamedDeclarationRoot, ...]: """Return validated top-level and grouped declaration roots for the requested kinds.""" - validate_named_declaration_layout(project_dir=project_dir) + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + if _LAYOUT_MEMO_KEY not in tree.memo: + validate_named_declaration_layout(project_dir=project_dir) + tree.memo[_LAYOUT_MEMO_KEY] = True roots: list[NamedDeclarationRoot] = [] for role_parts, kind in GLOBAL_NAMED_DECLARATION_ROOTS.items(): directory: Path = project_dir.joinpath(*role_parts) @@ -67,20 +74,22 @@ def named_declaration_roots( def named_declaration_files( - *, roots: tuple[NamedDeclarationRoot, ...], pattern: str + *, project_dir: Path, roots: tuple[NamedDeclarationRoot, ...], pattern: str ) -> Iterator[tuple[NamedDeclarationRoot, Path]]: """Yield every matching file below each root, paired with its root.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) root: NamedDeclarationRoot for root in roots: file_path: Path - for file_path in sorted(root.directory.rglob(pattern)): + for file_path in sorted(tree.rglob(root=root.directory, pattern=pattern)): yield root, file_path def validate_named_declaration_layout(*, project_dir: Path) -> None: """Reject entries that no named declaration role accepts.""" + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) audits_root: Path = project_dir / _AUDIT_ROLE_DIRECTORY if audits_root.is_dir(): _require_role_children( @@ -93,7 +102,7 @@ def validate_named_declaration_layout(*, project_dir: Path) -> None: directory: Path = project_dir / top_level if not directory.is_dir(): continue - for nested in sorted(path for path in directory.rglob("*") if path.is_dir()): + for nested in sorted(tree.directories(root=directory)): if nested.name in _NESTED_ROLE_DIRECTORIES: nested_path: str = _relative(path=nested, project_dir=project_dir) raise DeclarationParseError( @@ -154,6 +163,10 @@ def _require_role_children( def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Path], ...]: + tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + cached: object = tree.memo.get(_GROUPS_MEMO_KEY) + if cached is not None: + return cast(tuple[tuple[tuple[str, ...], Path], ...], cached) groups: list[tuple[tuple[str, ...], Path]] = [] for root_components in CANONICAL_AUTHORED_ROOTS: authored_root: Path = project_dir.joinpath(*root_components) @@ -161,10 +174,12 @@ def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Pa continue groups.extend( (root_components, group) - for group in sorted(authored_root.rglob(DECLARATION_GROUP_DIRECTORY)) + for group in sorted(tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY)) if group.is_dir() and group.parent != authored_root ) - return tuple(groups) + result: tuple[tuple[tuple[str, ...], Path], ...] = tuple(groups) + tree.memo[_GROUPS_MEMO_KEY] = result + return result def _relative(*, path: Path, project_dir: Path) -> str: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py new file mode 100644 index 0000000000..0db80edd8a --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py @@ -0,0 +1,68 @@ +"""Project-relative path facts shared by filesystem discovery.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from sqlbuild.compiler.discovery.constants import ( + CANONICAL_AUTHORED_ROOTS, + SQL_TESTS_OWNERSHIP_ROOT, +) +from sqlbuild.compiler.scopes.constants import ( + DECLARATION_GROUP_DIRECTORY, + SCOPED_DECLARATION_DIRECTORIES, +) +from sqlbuild.compiler.scopes.types import DeclarationKind + +_MACRO_TEST_DIRECTORY: str = f"{DeclarationKind.MACRO.value}s" +_SQL_FILE_SUFFIX: str = ".sql" +_SQL_TEST_ROOT_COMPONENTS: tuple[str, ...] = Path(SQL_TESTS_OWNERSHIP_ROOT).parts + + +def project_relative_path(*, path: Path, project_dir: Path) -> Path: + """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" + + root_parts: tuple[str, ...] = project_dir.parts + path_parts: tuple[str, ...] = path.parts + if ( + path.is_absolute() is not project_dir.is_absolute() + or path_parts[: len(root_parts)] != root_parts + ): + return path.relative_to(project_dir) + return path.with_segments(*path_parts[len(root_parts) :]) + + +def is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: + """Return whether a file lives inside a scoped declaration tree rather than a resource tree.""" + + return _is_scoped_declaration_directory( + directory=file_path.parent, + project_dir=project_dir, + sql_file=file_path.suffix == _SQL_FILE_SUFFIX, + ) + + +@lru_cache(maxsize=8192) +def _is_scoped_declaration_directory(*, directory: Path, project_dir: Path, sql_file: bool) -> bool: + """Classify a file's directory once; every file in it shares the same scoped-tree answer.""" + + directory_parts: tuple[str, ...] = project_relative_path( + path=directory, project_dir=project_dir + ).parts + root_components: tuple[str, ...] + for root_components in CANONICAL_AUTHORED_ROOTS: + if directory_parts[: len(root_components)] != root_components: + continue + scoped_components: tuple[str, ...] = directory_parts[len(root_components) :] + if ( + root_components == _SQL_TEST_ROOT_COMPONENTS + and sql_file + and scoped_components[:1] == (_MACRO_TEST_DIRECTORY,) + ): + return False + return any( + component == DECLARATION_GROUP_DIRECTORY or component in SCOPED_DECLARATION_DIRECTORIES + for component in scoped_components + ) + return False diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 117965b523..e9876628a6 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -23,6 +23,7 @@ ) from sqlbuild.compiler.discovery.models import ( ModelHeaderColumnSpan, + ModelHeaderMatch, NamedSqlHookEntry, PythonHookEntry, SqlHookEntry, @@ -115,10 +116,34 @@ def put(self, *, header: str, tokenization: _ModelHeaderTokenization) -> None: _MODEL_HEADER_TOKEN_CACHE: _ModelHeaderTokenCache = _ModelHeaderTokenCache() -def match_model_header(contents: str) -> re.Match[str] | None: +def match_model_header(contents: str) -> ModelHeaderMatch | None: """Match the MODEL(...) header once so callers can reuse it for every header projection.""" - return _MODEL_HEADER_PATTERN.match(contents) + header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) + if header_match is None: + return None + return ModelHeaderMatch( + contents=contents, + header_start=header_match.start("header"), + header_end=header_match.end("header"), + sql_start=header_match.start("sql"), + ) + + +def match_model_headers(contents: list[str]) -> list[ModelHeaderMatch | None]: + """Match many MODEL(...) headers natively with the same semantics as ``match_model_header``.""" + + return [ + None + if offsets is None + else ModelHeaderMatch( + contents=value, + header_start=offsets[0], + header_end=offsets[1], + sql_start=offsets[2], + ) + for value, offsets in zip(contents, _native.match_model_headers(contents), strict=True) + ] def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: @@ -128,7 +153,7 @@ def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object def parse_matched_model_sql( - *, header_match: re.Match[str] | None, file_path: Path + *, header_match: ModelHeaderMatch | None, file_path: Path ) -> tuple[dict[str, object], str]: """Parse header values and SQL body from one precomputed MODEL header match.""" @@ -139,7 +164,7 @@ def parse_matched_model_sql( ) header_values: dict[str, object] = parse_header_values( - header=header_match.group("header"), + header=header_match.header, file_path=file_path, statement_name="MODEL", ) @@ -153,12 +178,12 @@ def parse_matched_model_sql( header_values=header_values, supported_keys=SQL_MODEL_HEADER_KEYS, statement="MODEL()", - header=header_match.group("header"), - header_line=header_match.string.count("\n", 0, header_match.start("header")) + 1, + header=header_match.header, + header_line=header_match.contents.count("\n", 0, header_match.header_start) + 1, file_path=file_path, error_class=ModelSqlParseError, ) - query: str = header_match.group("sql").strip() + query: str = header_match.sql.strip() if not query: raise ModelSqlParseError(f"SQL model '{file_path}' must contain SQL after MODEL(...)") return header_values, query @@ -175,7 +200,7 @@ def model_header_column_locations( def matched_model_header_column_locations( - *, contents: str, header_match: re.Match[str] | None, relative_path: Path + *, contents: str, header_match: ModelHeaderMatch | None, relative_path: Path ) -> dict[str, SourceLocation]: """Return authored MODEL(columns) locations from one precomputed header match.""" @@ -183,8 +208,8 @@ def matched_model_header_column_locations( return {} return header_column_locations( contents=contents, - header=header_match.group("header"), - header_start=header_match.start("header"), + header=header_match.header, + header_start=header_match.header_start, relative_path=relative_path, ) @@ -314,7 +339,7 @@ def model_output_column_locations( def matched_model_output_column_locations( *, contents: str, - header_match: re.Match[str] | None, + header_match: ModelHeaderMatch | None, relative_path: Path, extract_implicit_alias_columns: bool, ) -> dict[str, SourceLocation]: @@ -322,8 +347,8 @@ def matched_model_output_column_locations( if header_match is None: return {} - sql_start: int = header_match.start("sql") - sql: str = header_match.group("sql") + sql_start: int = header_match.sql_start + sql: str = header_match.sql projection_ranges: tuple[tuple[int, int], ...] | None = _top_level_select_projection_ranges(sql) if projection_ranges is None: return {} @@ -674,15 +699,11 @@ def prepare_model_file_headers(contents: list[str]) -> None: prepare_matched_model_file_headers([match_model_header(value) for value in contents]) -def prepare_matched_model_file_headers(header_matches: list[re.Match[str] | None]) -> None: +def prepare_matched_model_file_headers(header_matches: list[ModelHeaderMatch | None]) -> None: """Batch tokenization for precomputed MODEL header matches.""" prepare_model_header_tokens( - [ - header_match.group("header") - for header_match in header_matches - if header_match is not None - ] + [header_match.header for header_match in header_matches if header_match is not None] ) diff --git a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py new file mode 100644 index 0000000000..fe030ff14d --- /dev/null +++ b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py @@ -0,0 +1,135 @@ +"""One shared directory snapshot for a discovery pass.""" + +from __future__ import annotations + +import fnmatch +import os +import re +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token +from functools import lru_cache +from pathlib import Path +from typing import ClassVar + +from sqlbuild.compiler.discovery.types import DirectorySnapshotEntry + +_CASE_PROBE: str = "A" +_DIRECTORIES_QUERY: str = "" + + +class DirectorySnapshot: + """Directory listings read once per discovery pass and shared by every glob in it.""" + + active: ClassVar[ContextVar[DirectorySnapshot | None]] = ContextVar( + "sqlbuild_directory_snapshot", default=None + ) + + def __init__(self, *, project_dir: Path) -> None: + self.project_dir: Path = project_dir + self.memo: dict[object, object] = {} + self._listings: dict[Path, tuple[DirectorySnapshotEntry, ...]] = {} + self._matches: dict[tuple[Path, str], tuple[Path, ...]] = {} + + @classmethod + @contextmanager + def scope(cls, *, project_dir: Path) -> Iterator[DirectorySnapshot]: + """Share one snapshot across every discovery query for the project inside the scope.""" + + active: DirectorySnapshot | None = cls.active.get() + if active is not None and active.project_dir == project_dir: + yield active + return + tree: DirectorySnapshot = DirectorySnapshot(project_dir=project_dir) + token: Token[DirectorySnapshot | None] = cls.active.set(tree) + try: + yield tree + finally: + cls.active.reset(token) + + @classmethod + def current(cls, *, project_dir: Path) -> DirectorySnapshot: + """Return the active snapshot for the project, or a single-use one outside a scope.""" + + active: DirectorySnapshot | None = cls.active.get() + if active is not None and active.project_dir == project_dir: + return active + return DirectorySnapshot(project_dir=project_dir) + + def rglob(self, *, root: Path, pattern: str) -> tuple[Path, ...]: + """Return the paths ``root.rglob(pattern)`` yields for a single-name pattern, unordered.""" + + match: Callable[[str], bool] = _name_matcher(pattern) + return self._select(root=root, key=pattern, accept=lambda entry: match(entry.name)) + + def directories(self, *, root: Path) -> tuple[Path, ...]: + """Return every directory below ``root``, including unwalked directory links, unordered.""" + + return self._select(root=root, key=_DIRECTORIES_QUERY, accept=lambda entry: entry.is_dir) + + def _select( + self, *, root: Path, key: str, accept: Callable[[DirectorySnapshotEntry], bool] + ) -> tuple[Path, ...]: + cached: tuple[Path, ...] | None = self._matches.get((root, key)) + if cached is not None: + return cached + selected: list[Path] = [] + if root.is_dir(): + directory: Path + for directory in self._walk(root=root): + selected.extend( + directory / entry.name + for entry in self._listing(directory=directory) + if accept(entry) + ) + result: tuple[Path, ...] = tuple(selected) + self._matches[(root, key)] = result + return result + + def _walk(self, *, root: Path) -> Iterator[Path]: + pending: list[Path] = [root] + while pending: + directory: Path = pending.pop() + yield directory + pending.extend( + directory / entry.name + for entry in self._listing(directory=directory) + if entry.is_walkable_dir + ) + + def _listing(self, *, directory: Path) -> tuple[DirectorySnapshotEntry, ...]: + cached: tuple[DirectorySnapshotEntry, ...] | None = self._listings.get(directory) + if cached is not None: + return cached + entries: list[DirectorySnapshotEntry] = [] + try: + with os.scandir(directory) as scanned: + entries.extend( + DirectorySnapshotEntry( + name=entry.name, + is_dir=_is_dir(entry=entry, follow_symlinks=True), + is_walkable_dir=_is_dir(entry=entry, follow_symlinks=False), + ) + for entry in scanned + ) + except OSError: + entries = [] + listing: tuple[DirectorySnapshotEntry, ...] = tuple(entries) + self._listings[directory] = listing + return listing + + +def _is_dir(*, entry: os.DirEntry[str], follow_symlinks: bool) -> bool: + try: + return entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + return False + + +@lru_cache(maxsize=64) +def _name_matcher(pattern: str) -> Callable[[str], bool]: + """Match names like ``Path.glob`` with the platform's default case sensitivity.""" + + flags: int = 0 if os.path.normcase(_CASE_PROBE) == _CASE_PROBE else re.IGNORECASE + compiled: re.Pattern[str] = re.compile(fnmatch.translate(pattern), flags) + return lambda name: compiled.match(name) is not None diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index a43a6f99f9..68453d746e 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -81,6 +81,28 @@ class DiscoveredSqlModelFile: extract_implicit_alias_columns: bool = True +@dataclass(frozen=True) +class ModelHeaderMatch: + """Offsets of one matched ``MODEL(...)`` header within its file contents.""" + + contents: str + header_start: int + header_end: int + sql_start: int + + @property + def header(self) -> str: + """Return the header body between the MODEL parentheses.""" + + return self.contents[self.header_start : self.header_end] + + @property + def sql(self) -> str: + """Return everything after the header terminator and following whitespace.""" + + return self.contents[self.sql_start :] + + @dataclass(frozen=True) class ModelHeaderColumnSpan: """Authored byte offsets for one MODEL(columns) entry and metadata body.""" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index cdc21e7b20..fc1896e66c 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -5,7 +5,7 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import Any, ClassVar, Protocol +from typing import Any, ClassVar, NamedTuple, Protocol from sqlbuild.compiler.scopes.types import ScopeKind @@ -25,3 +25,11 @@ class ScopedDeclarationFile(Protocol): ownership_root: Path | None owning_path: Path | None declaration_root: Path | None + + +class DirectorySnapshotEntry(NamedTuple): + """One directory entry in a discovery snapshot.""" + + name: str + is_dir: bool + is_walkable_dir: bool diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index f9d2453e32..4265621753 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.scopes._helpers.identities import format_identity from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes._helpers.visibility import build_visibility_index from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -116,6 +117,7 @@ def build_lookup(*, index: ScopeIndex) -> ScopeLookup: key=lambda item: item.resource, sort_key=grant_sort_key, ), + visibility_index=build_visibility_index(declarations=canonical_index.declarations), ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/visibility.py b/src/sqlbuild/compiler/scopes/_helpers/visibility.py index 6bed2d1e14..facd691eb2 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/visibility.py +++ b/src/sqlbuild/compiler/scopes/_helpers/visibility.py @@ -4,6 +4,7 @@ from functools import lru_cache from pathlib import PurePath +from types import MappingProxyType from sqlbuild.compiler.scopes._helpers.identities import parse_identity from sqlbuild.compiler.scopes._helpers.paths import normalize_path @@ -15,6 +16,8 @@ from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibility, + DeclarationVisibilityIndex, GrantRecord, InaccessibleRecord, OwnershipRoot, @@ -62,49 +65,186 @@ def query_target( def resolve_visibility( *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> VisibilityResolution: + query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) + declarations: tuple[DeclarationRecord, ...] = lookup.index.declarations + inaccessible: list[InaccessibleRecord] = [] + resource: ResourceRecord + positions: list[int] + for resource, positions in inaccessible_groups: + inaccessible.extend( + InaccessibleRecord( + resource.identity, + declarations[position].identity, + _inaccessible_reason(resource=resource, declaration=declarations[position]), + ) + for position in positions + ) + return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + + +def resolve_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + """Resolve visible facts and inaccessible identities without per-pair inaccessible reasons.""" + + query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) + identities: tuple[DeclarationIdentity, ...] = lookup.visibility_index.identities + inaccessible: list[DeclarationIdentity] = [] + positions: list[int] + for _resource, positions in inaccessible_groups: + inaccessible.extend(identities[position] for position in positions) + return DeclarationVisibility(query, tuple(visible), tuple(inaccessible)) + + +def _classify_target( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> tuple[ScopeTargetQuery, list[VisibilityRecord], list[tuple[ResourceRecord, list[int]]]]: + """Return the query, ordered visible records, and inaccessible positions per match.""" + query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - if query.unknown: - return VisibilityResolution(target=query) visible: list[VisibilityRecord] = [] - inaccessible: list[InaccessibleRecord] = [] + inaccessible_groups: list[tuple[ResourceRecord, list[int]]] = [] resource: ResourceRecord for resource in query.matches: - grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} - for grant in lookup.grants_by_resource.get(resource.identity, ()): - grants_by_declaration.setdefault(grant.declaration, []).append(grant) - declaration: DeclarationRecord - for declaration in lookup.index.declarations: - positive: VisibilityReason | None = _visibility_reason( - resource=resource, declaration=declaration + resource_visible, positions = _classify_resource(lookup=lookup, resource=resource) + visible.extend(resource_visible) + inaccessible_groups.append((resource, positions)) + return query, visible, inaccessible_groups + + +def _classify_resource( + *, lookup: ScopeLookup, resource: ResourceRecord +) -> tuple[list[VisibilityRecord], list[int]]: + """Return ordered visible records and the declaration positions left inaccessible.""" + + index: DeclarationVisibilityIndex = lookup.visibility_index + positive: list[tuple[int, VisibilityReason]] = _visible_positions( + index=index, resource=resource + ) + grants: tuple[GrantRecord, ...] = lookup.grants_by_resource.get(resource.identity, ()) + if not grants: + visible_positions: set[int] = {position for position, _reason in positive} + return ( + [ + VisibilityRecord(resource.identity, index.identities[position], reason) + for position, reason in positive + ], + [ + position + for position in range(len(index.identities)) + if position not in visible_positions + ], + ) + reasons: dict[int, VisibilityReason] = dict(positive) + grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} + for grant in grants: + grants_by_declaration.setdefault(grant.declaration, []).append(grant) + visible: list[VisibilityRecord] = [] + inaccessible: list[int] = [] + position: int + identity: DeclarationIdentity + for position, identity in enumerate(index.identities): + reason: VisibilityReason | None = reasons.get(position) + if reason is not None: + visible.append(VisibilityRecord(resource.identity, identity, reason)) + declaration_grants: list[GrantRecord] | None = grants_by_declaration.get(identity) + if declaration_grants is not None: + visible.extend( + VisibilityRecord( + resource.identity, + identity, + ( + VisibilityReason.TESTED_MACRO + if grant.kind is GrantKind.TESTED_MACRO + else VisibilityReason.EXPECTED_MODEL + ), + grant.through, + ) + for grant in declaration_grants ) - if positive is not None: - visible.append(VisibilityRecord(resource.identity, declaration.identity, positive)) - declaration_grants: list[GrantRecord] | None = grants_by_declaration.get( - declaration.identity + elif reason is None: + inaccessible.append(position) + return visible, inaccessible + + +def build_visibility_index( + *, declarations: tuple[DeclarationRecord, ...] +) -> DeclarationVisibilityIndex: + """Group canonical declaration positions by the scope facts that make them visible.""" + + global_positions: list[int] = [] + private: dict[ResourceIdentity, list[int]] = {} + local: dict[str, list[int]] = {} + inherited: dict[str, list[int]] = {} + position: int + declaration: DeclarationRecord + for position, declaration in enumerate(declarations): + if declaration.scope is ScopeKind.GLOBAL: + global_positions.append(position) + elif declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None: + private.setdefault(declaration.identity.owner, []).append(position) + elif declaration.scope is ScopeKind.LOCAL: + local.setdefault(_declaration_owner(declaration), []).append(position) + elif declaration.scope is ScopeKind.INHERITED: + inherited.setdefault(_declaration_owner(declaration), []).append(position) + return DeclarationVisibilityIndex( + identities=tuple(declaration.identity for declaration in declarations), + global_positions=tuple(global_positions), + private_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in private.items()} + ), + local_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in local.items()} + ), + inherited_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in inherited.items()} + ), + ) + + +def _visible_positions( + *, index: DeclarationVisibilityIndex, resource: ResourceRecord +) -> list[tuple[int, VisibilityReason]]: + """Return declaration positions visible to a resource, in declaration order.""" + + positive: list[tuple[int, VisibilityReason]] = [ + (position, VisibilityReason.GLOBAL) for position in index.global_positions + ] + positive.extend( + (position, VisibilityReason.PRIVATE_OWNER) + for position in index.private_positions.get(resource.identity, ()) + ) + if index.local_positions or index.inherited_positions: + parent: str = _parent(resource.path) + positive.extend( + (position, VisibilityReason.LOCAL_OWNER) + for position in index.local_positions.get(parent, ()) + ) + ancestor: str + for ancestor in _canonical_ancestors(parent): + positive.extend( + (position, VisibilityReason.INHERITED_ANCESTOR) + for position in index.inherited_positions.get(ancestor, ()) ) - if declaration_grants is not None: - visible.extend( - VisibilityRecord( - resource.identity, - declaration.identity, - ( - VisibilityReason.TESTED_MACRO - if grant.kind is GrantKind.TESTED_MACRO - else VisibilityReason.EXPECTED_MODEL - ), - grant.through, - ) - for grant in declaration_grants - ) - elif positive is None: - inaccessible.append( - InaccessibleRecord( - resource.identity, - declaration.identity, - _inaccessible_reason(resource=resource, declaration=declaration), - ) - ) - return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + positive.sort() + return positive + + +def _declaration_owner(declaration: DeclarationRecord) -> str: + return _canonical_owner(declaration.owning_path or CURRENT_PATH_COMPONENT) + + +@lru_cache(maxsize=65_536) +def _canonical_ancestors(path: str) -> tuple[str, ...]: + """Return every owner that ``_is_canonical_descendant`` accepts as an ancestor of ``path``.""" + + prefixes: list[str] = [CURRENT_PATH_COMPONENT] + index: int = path.find(PATH_SEPARATOR) + while index >= 0: + prefixes.append(path[:index]) + index = path.find(PATH_SEPARATOR, index + 1) + prefixes.append(path) + return tuple(dict.fromkeys(prefixes)) def resolve_path_visibility( diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index d57da707c8..0fdb41dd57 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -62,6 +62,9 @@ GLOBAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"macros", "enums", "constants"}) INHERITED_DECLARATION_DIRECTORIES: frozenset[str] = GLOBAL_DECLARATION_DIRECTORIES LOCAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"_macros", "_enums", "_constants"}) +SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( + INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES +) DECLARATION_DIRECTORY_FACTS: dict[str, tuple[DeclarationKind, ScopeKind]] = { "macros": (DeclarationKind.MACRO, ScopeKind.INHERITED), "enums": (DeclarationKind.ENUM, ScopeKind.INHERITED), diff --git a/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py new file mode 100644 index 0000000000..d4fe55c279 --- /dev/null +++ b/src/sqlbuild/compiler/scopes/main/_resolve_scope_declaration_visibility.py @@ -0,0 +1,16 @@ +"""Resolve static declaration visibility without per-declaration inaccessible reasons.""" + +from __future__ import annotations + +from pathlib import PurePath + +from sqlbuild.compiler.scopes._helpers.visibility import resolve_declaration_visibility +from sqlbuild.compiler.scopes.models import DeclarationVisibility, ResourceIdentity, ScopeLookup + + +def resolve_scope_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + """Return visible facts and inaccessible declaration identities for one target.""" + + return resolve_declaration_visibility(lookup=lookup, target=target) diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 6f46e02a7e..97ee535dd8 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -208,6 +208,17 @@ class ScopeIndex: completeness: ScopeCompleteness = field(default_factory=ScopeCompleteness) +@dataclass(frozen=True) +class DeclarationVisibilityIndex: + """Declaration positions grouped by the scope facts that make them visible.""" + + identities: tuple[DeclarationIdentity, ...] + global_positions: tuple[int, ...] + private_positions: Mapping[ResourceIdentity, tuple[int, ...]] + local_positions: Mapping[str, tuple[int, ...]] + inherited_positions: Mapping[str, tuple[int, ...]] + + @dataclass(frozen=True) class ScopeLookup: """Immutable indexes over one canonical ``ScopeIndex``.""" @@ -219,6 +230,7 @@ class ScopeLookup: usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] + visibility_index: DeclarationVisibilityIndex @dataclass(frozen=True) @@ -245,6 +257,15 @@ class VisibilityResolution: inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class DeclarationVisibility: + """Visible facts and inaccessible declaration identities for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) + inaccessible: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) + + @dataclass(frozen=True) class ScopeReportFilters: """Orthogonal declaration filters applied in their documented fixed order.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py new file mode 100644 index 0000000000..8f6eddcf43 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_helpers.py @@ -0,0 +1,87 @@ +"""Warm compile target fixtures and artifact inspection for target reconciliation tests.""" + +from __future__ import annotations + +import os +from itertools import compress +from pathlib import Path + +from sqlbuild.cli.commands.main.entrypoint.entry import main + +_WARM_TARGET_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "shop"\nadapter = "duckdb"\n', + "models/sales/orders.sql": ( + "MODEL (description 'Test model orders.', columns (order_id (type INTEGER, " + "audits [not_null])));\nSELECT 1 AS order_id\n" + ), + "models/sales/daily/totals.sql": ( + "MODEL (description 'Test model totals.');\nSELECT order_id FROM __ref(\"orders\")\n" + ), + "models/inventory.sql": "MODEL (description 'Test model inventory.');\nSELECT 2 AS item_id\n", +} + + +def write_warm_target_project(*, project_dir: Path) -> None: + """Write a small project with nested models and one attached audit.""" + + for relative_path, contents in _WARM_TARGET_PROJECT_FILES.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def compile_project(*, project_dir: Path, compile_args: tuple[str, ...]) -> int: + """Run the real compile command and return its exit code.""" + + return main(["--project-dir", str(project_dir), "--no-color", "compile", *compile_args]) + + +def compiled_artifacts(*, compiled_dir: Path) -> tuple[Path, ...]: + """Return every regular file below target/compiled, without following directory links.""" + + artifacts: list[Path] = [] + for root, _directories, names in os.walk(compiled_dir): + artifacts.extend(Path(root, name) for name in names) + return tuple(sorted(artifacts)) + + +def stamp_artifacts(*, artifacts: tuple[Path, ...], mtime_ns: int) -> None: + """Give every artifact one known modification time.""" + + for path in artifacts: + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def changed_artifacts(*, compiled_dir: Path, mtime_ns: int) -> tuple[str, ...]: + """Return artifacts whose modification time moved away from the stamped value.""" + + artifacts: tuple[Path, ...] = compiled_artifacts(compiled_dir=compiled_dir) + changed: list[bool] = [path.stat().st_mtime_ns != mtime_ns for path in artifacts] + return tuple(path.relative_to(compiled_dir).as_posix() for path in compress(artifacts, changed)) + + +def pruned_entries(*, entries: dict[str, Path], names: tuple[str, ...]) -> tuple[str, ...]: + """Return the named stale entries that no longer exist.""" + + return tuple(compress(names, [not entries[name].exists() for name in names])) + + +def write_stale_target_entries(*, compiled_dir: Path, outside_dir: Path) -> dict[str, Path]: + """Add a stale nested artifact, an empty folder, and a link to an outside folder.""" + + stale_file: Path = compiled_dir / "models" / "legacy" / "archive" / "returns.sql" + stale_file.parent.mkdir(parents=True) + stale_file.write_text("SELECT 'stale'\n", encoding="utf-8") + empty_directory: Path = compiled_dir / "models" / "empty" / "nested" + empty_directory.mkdir(parents=True) + outside_dir.mkdir() + (outside_dir / "kept.sql").write_text("SELECT 'kept'\n", encoding="utf-8") + link: Path = compiled_dir / "models" / "linked" / "shared" + link.parent.mkdir(parents=True) + link.symlink_to(outside_dir, target_is_directory=True) + return { + "legacy": compiled_dir / "models" / "legacy", + "empty": compiled_dir / "models" / "empty", + "link": link, + "outside_file": outside_dir / "kept.sql", + } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index af6a8c559d..4bf59b42a7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -910,3 +910,22 @@ class UnusedOutputDifferentialCase: cte_sql: str reader_sql: str expected_statuses: tuple[str, ...] + + +@dataclass(frozen=True) +class WarmTargetTestCase: + description: str + compile_args: tuple[str, ...] + expected_exit_code: int = 0 + expected_changed: tuple[str, ...] = () + expected_pruned: tuple[str, ...] = ("legacy", "empty") + expected_link_kept: bool = True + + +@dataclass(frozen=True) +class WarmTargetEditTestCase: + description: str + edited_path: str + edited_contents: str + expected_changed: tuple[str, ...] + expected_exit_code: int = 0 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py new file mode 100644 index 0000000000..6af6a5fbb1 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py @@ -0,0 +1,100 @@ +"""End-to-end coverage for warm compile target reconciliation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_helpers import ( + changed_artifacts, + compile_project, + compiled_artifacts, + pruned_entries, + stamp_artifacts, + write_stale_target_entries, + write_warm_target_project, +) +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + WarmTargetEditTestCase, + WarmTargetTestCase, +) + +_UNCHANGED_MTIME_NS: int = 1_000_000_000 + + +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetTestCase("compile_cache_enabled", compile_args=()), + WarmTargetTestCase("compile_cache_disabled", compile_args=("--no-cache",)), + ], + ids=lambda case: case.description, +) +def test_given_warm_target_with_stale_entries_when_compiling_then_preserves_unchanged_and_prunes( + tmp_path: Path, capsys: pytest.CaptureFixture[str], test_case: WarmTargetTestCase +) -> None: + project_dir: Path = tmp_path / "project" + write_warm_target_project(project_dir=project_dir) + first_exit_code: int = compile_project( + project_dir=project_dir, compile_args=test_case.compile_args + ) + compiled_dir: Path = project_dir / "target" / "compiled" + managed: tuple[Path, ...] = compiled_artifacts(compiled_dir=compiled_dir) + stamp_artifacts(artifacts=managed, mtime_ns=_UNCHANGED_MTIME_NS) + stale: dict[str, Path] = write_stale_target_entries( + compiled_dir=compiled_dir, outside_dir=tmp_path / "shared" + ) + + second_exit_code: int = compile_project( + project_dir=project_dir, compile_args=test_case.compile_args + ) + _ = capsys.readouterr() + + assert (first_exit_code, second_exit_code) == ( + test_case.expected_exit_code, + test_case.expected_exit_code, + ) + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_changed + ) + assert compiled_artifacts(compiled_dir=compiled_dir) == managed + assert pruned_entries(entries=stale, names=("legacy", "empty")) == test_case.expected_pruned + assert stale["link"].is_symlink() is test_case.expected_link_kept + assert stale["outside_file"].is_file() is test_case.expected_link_kept + + +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetEditTestCase( + "one_model_edited", + edited_path="models/inventory.sql", + edited_contents="MODEL (description 'Test model inventory.');\nSELECT 3 AS item_id\n", + expected_changed=("models/inventory.sql",), + ) + ], + ids=lambda case: case.description, +) +def test_given_one_edited_model_when_compiling_warm_then_rewrites_only_its_artifacts( + tmp_path: Path, capsys: pytest.CaptureFixture[str], test_case: WarmTargetEditTestCase +) -> None: + write_warm_target_project(project_dir=tmp_path) + _ = compile_project(project_dir=tmp_path, compile_args=()) + compiled_dir: Path = tmp_path / "target" / "compiled" + stamp_artifacts( + artifacts=compiled_artifacts(compiled_dir=compiled_dir), mtime_ns=_UNCHANGED_MTIME_NS + ) + (tmp_path / test_case.edited_path).write_text(test_case.edited_contents, encoding="utf-8") + + exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + _ = capsys.readouterr() + + assert exit_code == test_case.expected_exit_code + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_changed + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py new file mode 100644 index 0000000000..200495ee9a --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -0,0 +1,217 @@ +"""Randomized trees, projects, and model files for shared-snapshot discovery tests.""" + +from __future__ import annotations + +import random +import re +from collections.abc import Callable +from itertools import compress +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem import aggregation +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_audit_files, + discover_constant_files, + discover_enum_files, + discover_macro_files, + discover_model_files, + discover_model_schema_files, + discover_schema_files, + discover_seed_files, + discover_sql_function_files, + discover_sql_hook_files, + discover_test_files, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import ( + match_model_header, + match_model_headers, +) +from sqlbuild.compiler.discovery._helpers.yml.project import ( + load_local_config, + load_project_config, +) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + +_DIRECTORY_NAMES: tuple[str, ...] = ( + "orders", + "customers", + "_sqlbuild", + "enums", + "_enums", + "macros", + ".hidden", +) +_FILE_NAMES: tuple[str, ...] = ( + "schema.yml", + "daily.sql", + "daily.SQL", + "helper.py", + "notes.yml", + ".draft.sql", +) +_LINK_NAMES: tuple[str, ...] = ("shared", "enums_link", "_sqlbuild_link") +_PATTERNS: tuple[str, ...] = ("*", "*.sql", "*.py", "*.yml", "schema.yml", "_sqlbuild", "enums") +_ISOLATED_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { + "model_files": discover_model_files, + "enum_files": discover_enum_files, + "constant_files": discover_constant_files, + "model_schema_files": discover_model_schema_files, + "sql_function_files": discover_sql_function_files, + "sql_hook_files": discover_sql_hook_files, + "schema_files": discover_schema_files, + "seed_files": discover_seed_files, + "test_files": discover_test_files, + "audit_files": discover_audit_files, + "macro_files": discover_macro_files, +} +_SCOPED_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "shop"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "models/sales/daily.sql": "MODEL (description 'Daily.');\nSELECT 1 AS day_id\n", + "models/sales/schema.yml": "models: []\n", + "models/sales/enums/region.sql": "ENUM (name region, members [NORTH, SOUTH]);\n", + "models/sales/_sqlbuild/constants/limit.sql": "CONSTANT (name order_limit, value 10);\n", + "models/sales/_sqlbuild/audits/generic/positive.sql": ( + "AUDIT (name positive);\nSELECT 1 WHERE FALSE\n" + ), + "macros/money.py": "def money(value):\n return value\n", + "enums/status.sql": "ENUM (name status, members [OPEN, CLOSED]);\n", + "functions/sql/tax.sql": ( + "FUNCTION (arguments (amount DOUBLE), returns DOUBLE);\namount * 0.2\n" + ), +} +_HEADER_FRAGMENTS: tuple[str, ...] = ( + "MODEL", + "model", + "(", + ")", + ";", + "'", + '"', + "\\", + "name orders", + "columns (id INT)", + "SELECT 1", + "\n", + " ", + "é", + "\u00a0", + "\u3000", + "\u2028", + "\x1c", + "\U0001f600", +) +_PYTHON_WHITESPACE: tuple[str, ...] = tuple(re.findall(r"\s", "".join(map(chr, range(0x3001))))) + + +def write_random_tree(*, root: Path, seed: int, entry_count: int) -> None: + """Write a seeded tree of folders, files, hidden entries, and directory links.""" + + rng: random.Random = random.Random(seed) + directories: list[Path] = [root] + creators: tuple[Callable[[random.Random, Path, list[Path]], None], ...] = ( + *(_add_directory,) * 9, + *(_add_file,) * 9, + *(_add_link,) * 2, + ) + for _index in range(entry_count): + rng.choice(creators)(rng, rng.choice(directories), directories) + + +def snapshot_glob_mismatches(*, root: Path) -> tuple[str, ...]: + """Return every root and pattern where the snapshot disagrees with ``Path.rglob``.""" + + tree: DirectorySnapshot = DirectorySnapshot(project_dir=root) + directories: list[Path] = [root, *sorted(filter(Path.is_dir, root.rglob("*")))] + labels: list[str] = [] + matches: list[bool] = [] + for directory in directories: + for pattern in _PATTERNS: + labels.append(f"{directory}:{pattern}") + matches.append( + sorted(tree.rglob(root=directory, pattern=pattern)) + == sorted(directory.rglob(pattern)) + ) + labels.append(f"{directory}:") + matches.append( + sorted(tree.directories(root=directory)) + == sorted(filter(Path.is_dir, directory.rglob("*"))) + ) + labels.append("absent root") + matches.append(tree.rglob(root=root / "absent", pattern="*") == ()) + return tuple(compress(labels, [not matched for matched in matches])) + + +def write_scoped_project(*, root: Path) -> None: + """Write a project with resources, scoped and grouped declarations, and global roles.""" + + for relative_path, contents in _SCOPED_PROJECT_FILES.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def isolated_discovery_mismatches(*, project_dir: Path) -> tuple[str, ...]: + """Return categories where one shared-snapshot pass differs from isolated discovery.""" + + inputs: DiscoveredProjectInputs = aggregation.build_discovered_project_inputs( + project_dir=project_dir, + project_config=load_project_config(project_dir=project_dir), + local_config=load_local_config(project_dir=project_dir), + sql_analysis_enabled=True, + ) + return tuple( + compress( + _ISOLATED_DISCOVERERS, + [ + getattr(inputs, name) != discover(project_dir=project_dir) + for name, discover in _ISOLATED_DISCOVERERS.items() + ], + ) + ) + + +def model_paths(*, inputs: DiscoveredProjectInputs) -> tuple[str, ...]: + """Return discovered model paths in discovery order.""" + + return tuple(file.relative_path.as_posix() for file in inputs.model_files) + + +def header_match_mismatches(*, seed: int, count: int) -> tuple[str, ...]: + """Return random MODEL files whose native header match differs from the Python pattern.""" + + rng: random.Random = random.Random(seed) + contents: list[str] = [_random_model_file(rng) for _ in range(count)] + contents.extend(f"MODEL (a){space};{space}SELECT 1" for space in _PYTHON_WHITESPACE) + return tuple( + compress( + contents, + [ + native != match_model_header(value) + for value, native in zip(contents, match_model_headers(contents), strict=True) + ], + ) + ) + + +def _random_model_file(rng: random.Random) -> str: + prefix: str = "".join(rng.choice(_PYTHON_WHITESPACE) for _ in range(rng.randint(0, 2))) + body: str = "".join(rng.choice(_HEADER_FRAGMENTS) for _ in range(rng.randint(0, 14))) + return f"{prefix}MODEL{rng.choice(_PYTHON_WHITESPACE)}({body}" + + +def _add_directory(rng: random.Random, parent: Path, directories: list[Path]) -> None: + path: Path = parent / rng.choice(_DIRECTORY_NAMES) + path.mkdir(exist_ok=True) + directories.append(path) + + +def _add_file(rng: random.Random, parent: Path, _directories: list[Path]) -> None: + (parent / rng.choice(_FILE_NAMES)).write_text("SELECT 1\n", encoding="utf-8") + + +def _add_link(rng: random.Random, parent: Path, directories: list[Path]) -> None: + link: Path = parent / rng.choice(_LINK_NAMES) + link.unlink(missing_ok=True) + link.symlink_to(rng.choice([*directories, parent / "missing"]), target_is_directory=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 0425fc326c..c85fe6b18e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -940,3 +940,42 @@ class StatementHeaderKeysTestCase: contents: str expected_error: str expected_help: str + + +@dataclass(frozen=True) +class SnapshotGlobTestCase: + description: str + seed: int + entry_count: int + expected_mismatches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SharedSnapshotDiscoveryTestCase: + description: str + expected_model_paths: tuple[str, ...] + expected_mismatches: tuple[str, ...] = () + + +@dataclass(frozen=True) +class DiscoveryPassRefreshTestCase: + description: str + added_files: dict[str, str] + removed_path: str + expected_first_paths: tuple[str, ...] + expected_second_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class SnapshotScopeReuseTestCase: + description: str + expected_nested_reuse: bool + expected_later_reuse: bool + + +@dataclass(frozen=True) +class NativeHeaderMatchTestCase: + description: str + seed: int + count: int + expected_mismatches: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py new file mode 100644 index 0000000000..12c88af4de --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_directory_snapshot.py @@ -0,0 +1,147 @@ +"""Tests for the shared single-walk discovery snapshot.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_helpers import ( + header_match_mismatches, + isolated_discovery_mismatches, + model_paths, + snapshot_glob_mismatches, + write_random_tree, + write_scoped_project, +) +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + DiscoveryPassRefreshTestCase, + NativeHeaderMatchTestCase, + SharedSnapshotDiscoveryTestCase, + SnapshotGlobTestCase, + SnapshotScopeReuseTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotGlobTestCase("small_tree", seed=1, entry_count=40), + SnapshotGlobTestCase("medium_tree", seed=2, entry_count=120), + SnapshotGlobTestCase("large_tree", seed=3, entry_count=300), + ], + ids=lambda case: case.description, +) +def test_given_randomized_tree_when_globbing_snapshot_then_matches_pathlib_rglob( + tmp_path: Path, test_case: SnapshotGlobTestCase +) -> None: + write_random_tree(root=tmp_path, seed=test_case.seed, entry_count=test_case.entry_count) + + mismatches: tuple[str, ...] = snapshot_glob_mismatches(root=tmp_path) + + assert mismatches == test_case.expected_mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + SharedSnapshotDiscoveryTestCase( + "scoped_and_grouped_declarations", + expected_model_paths=("models/orders.sql", "models/sales/daily.sql"), + ) + ], + ids=lambda case: case.description, +) +def test_given_shared_snapshot_when_discovering_project_then_matches_isolated_discovery( + tmp_path: Path, test_case: SharedSnapshotDiscoveryTestCase +) -> None: + write_scoped_project(root=tmp_path) + + mismatches: tuple[str, ...] = isolated_discovery_mismatches(project_dir=tmp_path) + + assert mismatches == test_case.expected_mismatches + assert model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) == ( + test_case.expected_model_paths + ) + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoveryPassRefreshTestCase( + "added_and_removed_models", + added_files={ + "models/sales/weekly.sql": "MODEL (description 'Weekly.');\nSELECT 1 AS week_id\n" + }, + removed_path="models/orders.sql", + expected_first_paths=("models/orders.sql", "models/sales/daily.sql"), + expected_second_paths=("models/sales/daily.sql", "models/sales/weekly.sql"), + ) + ], + ids=lambda case: case.description, +) +def test_given_files_change_between_discovery_passes_when_discovering_then_sees_current_tree( + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], + test_case: DiscoveryPassRefreshTestCase, +) -> None: + write_scoped_project(root=tmp_path) + first: tuple[str, ...] = model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) + write_repo_files(tmp_path, test_case.added_files) + (tmp_path / test_case.removed_path).unlink() + + second: tuple[str, ...] = model_paths(inputs=discover_project_inputs(project_dir=tmp_path)) + + assert first == test_case.expected_first_paths + assert second == test_case.expected_second_paths + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScopeReuseTestCase( + "nested_reuses_later_refreshes", + expected_nested_reuse=True, + expected_later_reuse=False, + ) + ], + ids=lambda case: case.description, +) +def test_given_nested_scope_for_same_project_when_entering_then_reuses_active_snapshot( + tmp_path: Path, test_case: SnapshotScopeReuseTestCase +) -> None: + with DirectorySnapshot.scope(project_dir=tmp_path) as outer: + with DirectorySnapshot.scope(project_dir=tmp_path) as inner: + nested_reuse: bool = inner is outer + with DirectorySnapshot.scope(project_dir=tmp_path) as later: + later_reuse: bool = later is outer + + assert nested_reuse is test_case.expected_nested_reuse + assert later_reuse is test_case.expected_later_reuse + + +@pytest.mark.parametrize( + "test_case", + [ + NativeHeaderMatchTestCase("seed_1", seed=1, count=2000), + NativeHeaderMatchTestCase("seed_2", seed=2, count=2000), + NativeHeaderMatchTestCase("seed_3", seed=3, count=2000), + NativeHeaderMatchTestCase("seed_4", seed=4, count=2000), + ], + ids=lambda case: case.description, +) +def test_given_randomized_model_files_when_matching_natively_then_matches_python_pattern( + test_case: NativeHeaderMatchTestCase, +) -> None: + mismatches: tuple[str, ...] = header_match_mismatches( + seed=test_case.seed, count=test_case.count + ) + + assert mismatches == test_case.expected_mismatches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py new file mode 100644 index 0000000000..99712b3c65 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -0,0 +1,260 @@ +"""Reference visibility classification and randomized scope projects for equivalence tests.""" + +from __future__ import annotations + +import posixpath +import random +from collections.abc import Callable +from itertools import compress +from pathlib import PurePath + +from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes._helpers.visibility import query_target +from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( + resolve_scope_declaration_visibility, +) +from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + DeclarationVisibility, + GrantRecord, + InaccessibleRecord, + OwnershipRoot, + ResourceIdentity, + ResourceRecord, + ScopeIndex, + ScopeLookup, + ScopeTargetQuery, + VisibilityRecord, + VisibilityResolution, +) +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + GrantKind, + InaccessibleReason, + ResourceKind, + ScopeKind, + VisibilityReason, +) + +_FOLDERS: tuple[str, ...] = ("sales", "sale", "finance", "daily", "orders") +_ROOTS: tuple[str, ...] = ("models", "tests/unit", "functions/sql") +_NAMES: tuple[str, ...] = ("status", "region", "tier", "limit", "currency", "channel") +_NOISY_SPELLINGS: tuple[Callable[[random.Random, str], str], ...] = ( + lambda _rng, path: path.replace("/", "\\"), + lambda _rng, path: f"./{path}/", + lambda rng, path: f"{path}/{rng.choice(_FOLDERS)}/..", + lambda _rng, path: path, +) +_VISIBLE_REASONS: dict[ScopeKind, VisibilityReason] = { + ScopeKind.GLOBAL: VisibilityReason.GLOBAL, + ScopeKind.PRIVATE: VisibilityReason.PRIVATE_OWNER, + ScopeKind.LOCAL: VisibilityReason.LOCAL_OWNER, + ScopeKind.INHERITED: VisibilityReason.INHERITED_ANCESTOR, +} +_GRANT_REASONS: dict[GrantKind, VisibilityReason] = { + kind: VisibilityReason.EXPECTED_MODEL for kind in GrantKind +} | {GrantKind.TESTED_MACRO: VisibilityReason.TESTED_MACRO} + + +def random_scope_lookup( + *, seed: int, resource_count: int, declaration_count: int, grant_count: int +) -> ScopeLookup: + """Build a seeded project with every scope tier, noisy owner paths, and grants.""" + + rng: random.Random = random.Random(seed) + resources: list[ResourceRecord] = [] + for index in range(resource_count): + directory: str = _directory(rng) + name: str = f"model_{index % max(1, resource_count - 3)}" + resources.append( + ResourceRecord( + ResourceIdentity(ResourceKind.MODEL, name), + f"{directory}/{name}.sql", + OwnershipRoot(directory.split("/")[0], resource_kind=ResourceKind.MODEL), + ) + ) + identities: list[ResourceIdentity] = [resource.identity for resource in resources] + owners: list[ResourceIdentity] = [*identities, ResourceIdentity(ResourceKind.MODEL, "other")] + declarations: list[DeclarationRecord] = [] + for _index in range(declaration_count): + scope: ScopeKind = rng.choice(tuple(ScopeKind)) + owner_directory: str = _directory(rng) + owning_path: str = rng.choice(_NOISY_SPELLINGS)(rng, owner_directory) + declarations.append( + DeclarationRecord( + DeclarationIdentity( + rng.choice((DeclarationKind.ENUM, DeclarationKind.MACRO)), + rng.choice(_NAMES), + {ScopeKind.PRIVATE: rng.choice(owners)}.get(scope), + ), + f"{owner_directory}/_sqlbuild/enums/{rng.randint(0, 9)}.sql", + rng.randint(1, 5), + 1, + scope, + OwnershipRoot(owner_directory.split("/")[0], resource_kind=ResourceKind.MODEL), + owning_path={True: None, False: owning_path}[ + scope is ScopeKind.GLOBAL or rng.random() < 0.1 + ], + ) + ) + grants: list[GrantRecord] = [ + GrantRecord( + rng.choice(identities), + rng.choice(declarations).identity, + rng.choice(identities), + rng.choice(tuple(GrantKind)), + ) + for _index in range(grant_count * bool(declarations and identities)) + ] + return build_scope_lookup( + index=ScopeIndex( + resources=tuple(resources), declarations=tuple(declarations), grants=tuple(grants) + ) + ) + + +def visibility_mismatches(*, lookup: ScopeLookup) -> tuple[str, ...]: + """Return every target whose indexed resolution differs from the exhaustive reference.""" + + targets: list[ResourceIdentity | str] = _targets(lookup) + differs: list[bool] = [ + resolve_scope_visibility(lookup=lookup, target=target) + != _reference_resolution(lookup=lookup, target=target) + for target in targets + ] + return tuple(map(str, compress(targets, differs))) + + +def declaration_visibility_mismatches(*, lookup: ScopeLookup) -> tuple[str, ...]: + """Return every target whose reason-free resolution differs from the exhaustive reference.""" + + targets: list[ResourceIdentity | str] = _targets(lookup) + differs: list[bool] = [ + resolve_scope_declaration_visibility(lookup=lookup, target=target) + != _reference_declaration_visibility(lookup=lookup, target=target) + for target in targets + ] + return tuple(map(str, compress(targets, differs))) + + +def _directory(rng: random.Random) -> str: + root: str = rng.choice(_ROOTS) + depth: int = rng.randint(0, 3) + return "/".join([root, *(rng.choice(_FOLDERS) for _ in range(depth))]) + + +def _targets(lookup: ScopeLookup) -> list[ResourceIdentity | str]: + return [ + *lookup.resources, + *(resource.path for resource in lookup.index.resources), + "models/missing.sql", + "model:missing", + ] + + +def _parent(path: str) -> str: + return posixpath.dirname(normalize_path(path=path)) or "." + + +def _is_descendant(*, path: str, ancestor: str) -> bool: + return ancestor == "." or path == ancestor or path.startswith(f"{ancestor}/") + + +def _reference_reason( + *, resource: ResourceRecord, declaration: DeclarationRecord +) -> VisibilityReason | None: + owner: str = normalize_path(path=declaration.owning_path or ".") + parent: str = _parent(resource.path) + visible: dict[ScopeKind, bool] = { + ScopeKind.GLOBAL: True, + ScopeKind.PRIVATE: declaration.identity.owner == resource.identity, + ScopeKind.LOCAL: parent == owner, + ScopeKind.INHERITED: _is_descendant(path=parent, ancestor=owner), + } + return {True: _VISIBLE_REASONS[declaration.scope], False: None}[visible[declaration.scope]] + + +def _reference_inaccessible_reason( + *, resource: ResourceRecord, declaration: DeclarationRecord +) -> InaccessibleReason: + owner: str = normalize_path(path=declaration.owning_path or ".") + candidates: tuple[tuple[bool, InaccessibleReason], ...] = ( + (declaration.scope is ScopeKind.PRIVATE, InaccessibleReason.PRIVATE_OWNER), + (declaration.scope is ScopeKind.LOCAL, InaccessibleReason.LOCAL_BOUNDARY), + ( + _is_descendant(path=owner, ancestor=_parent(resource.path)), + InaccessibleReason.DESCENDANT_SCOPE, + ), + ( + declaration.ownership_root.path == resource.ownership_root.path, + InaccessibleReason.SIBLING_SCOPE, + ), + (True, InaccessibleReason.UNRELATED_SCOPE), + ) + return next( + compress( + (reason for _matches, reason in candidates), + (matches for matches, _reason in candidates), + ) + ) + + +def _reference_resolution( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> VisibilityResolution: + """Classify every declaration against every matched resource, as before indexing.""" + + query: ScopeTargetQuery = query_target(lookup=lookup, target=target) + visible: list[VisibilityRecord] = [] + inaccessible: list[InaccessibleRecord] = [] + for resource in query.matches: + grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} + for grant in lookup.grants_by_resource.get(resource.identity, ()): + grants_by_declaration.setdefault(grant.declaration, []).append(grant) + for declaration in lookup.index.declarations: + positive: VisibilityReason | None = _reference_reason( + resource=resource, declaration=declaration + ) + granted: list[GrantRecord] | None = grants_by_declaration.get(declaration.identity) + visible.extend( + VisibilityRecord(resource.identity, declaration.identity, reason) + for reason in filter(None, (positive,)) + ) + visible.extend( + VisibilityRecord( + resource.identity, + declaration.identity, + _GRANT_REASONS[grant.kind], + grant.through, + ) + for grant in granted or () + ) + inaccessible.extend( + compress( + [ + InaccessibleRecord( + resource.identity, + declaration.identity, + _reference_inaccessible_reason( + resource=resource, declaration=declaration + ), + ) + ], + [granted is None and positive is None], + ) + ) + return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) + + +def _reference_declaration_visibility( + *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath +) -> DeclarationVisibility: + expected: VisibilityResolution = _reference_resolution(lookup=lookup, target=target) + return DeclarationVisibility( + expected.target, + expected.visible, + tuple(record.declaration for record in expected.inaccessible), + ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 785f725133..feea050209 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -229,3 +229,15 @@ class RelocationCase: consumer_paths: tuple[str, ...] destination: str expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class RandomScopeCase: + """One seeded randomized scope project compared with the exhaustive classification.""" + + description: str + seed: int + resource_count: int + declaration_count: int + grant_count: int + expected_mismatches: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py b/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py new file mode 100644 index 0000000000..c9452910f6 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_visibility_index.py @@ -0,0 +1,74 @@ +"""Equivalence of indexed scope visibility with the exhaustive per-declaration classification.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.compiler.scopes._test_helpers import ( + declaration_visibility_mismatches, + random_scope_lookup, + visibility_mismatches, +) +from tests.unit.src.sqlbuild.compiler.scopes._test_types import RandomScopeCase + + +@pytest.mark.parametrize( + "test_case", + [ + RandomScopeCase("sparse", seed=1, resource_count=12, declaration_count=20, grant_count=0), + RandomScopeCase("mixed", seed=2, resource_count=30, declaration_count=60, grant_count=6), + RandomScopeCase("dense", seed=3, resource_count=50, declaration_count=120, grant_count=15), + RandomScopeCase( + "no_declarations", seed=4, resource_count=8, declaration_count=0, grant_count=0 + ), + RandomScopeCase( + "grant_heavy", seed=5, resource_count=40, declaration_count=80, grant_count=40 + ), + RandomScopeCase("wide", seed=6, resource_count=25, declaration_count=200, grant_count=3), + ], + ids=lambda case: case.description, +) +def test_given_randomized_scopes_when_resolving_visibility_then_matches_exhaustive_classification( + test_case: RandomScopeCase, +) -> None: + mismatches: tuple[str, ...] = visibility_mismatches( + lookup=random_scope_lookup( + seed=test_case.seed, + resource_count=test_case.resource_count, + declaration_count=test_case.declaration_count, + grant_count=test_case.grant_count, + ) + ) + + assert mismatches == test_case.expected_mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + RandomScopeCase("sparse", seed=1, resource_count=12, declaration_count=20, grant_count=0), + RandomScopeCase("mixed", seed=2, resource_count=30, declaration_count=60, grant_count=6), + RandomScopeCase( + "grant_heavy", seed=5, resource_count=40, declaration_count=80, grant_count=40 + ), + RandomScopeCase("wide", seed=6, resource_count=25, declaration_count=200, grant_count=3), + ], + ids=lambda case: case.description, +) +def test_given_randomized_scopes_when_resolving_declarations_then_matches_exhaustive_identities( + test_case: RandomScopeCase, +) -> None: + mismatches: tuple[str, ...] = declaration_visibility_mismatches( + lookup=random_scope_lookup( + seed=test_case.seed, + resource_count=test_case.resource_count, + declaration_count=test_case.declaration_count, + grant_count=test_case.grant_count, + ) + ) + + assert mismatches == test_case.expected_mismatches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index b9417cff64..0cef3bee65 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -42,7 +42,10 @@ line_width = 100 `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; `sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it before compile does (see [Descriptions](/docs/concepts/models/configuration#descriptions)). A model -described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +described by its `model_schema`, or by the one `[path_defaults]` entry that applies to it, passes, +exactly as it compiles. `[defaults]` does not accept `description` (`D001`), and only the nearest +matching path default applies (see +[Which path default applies](/docs/concepts/project-configuration#which-path-default-applies)). Format does not render templates, so a templated description that renders empty passes format and is reported by `sqb compile`. Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 8a888be1b1..8cffb29913 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -363,14 +363,52 @@ replay_on_change = "full" ``` Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the -`staging` path default. +`staging` path default. A key matches its folder and every folder below it. In a key, `*` matches one +folder name and `**` matches any number of folders, so `"**"` matches every model. + +### Which path default applies + +Each model uses at most one path default: the nearest matching key. Path defaults are not merged, so +a model gets nothing from a broader key when a more specific key also matches it. + +1. A key without wildcards beats every wildcard key. Among those, the deepest key wins, so + `staging/orders` beats `staging`. +2. When only wildcard keys match, the most specific wins: more literal folder names first, then more + `*` segments, then fewer `**` segments. Two equally specific matches fail compilation. + +A broad `"**"` default therefore applies only to models that no other key matches. With the +configuration below, `models/orders.sql` gets the `"**"` description, but +`models/staging/stg_customers.sql` uses only the `staging` entry. It has no description and fails +with `P010`: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +``` + +Descriptions belong to individual models, so put each one in the `MODEL()` header, or in the +`description` of the model's `model_schema`. `[defaults]` rejects `description` with `D001`. If a +folder of models really shares one description, every key those models can select must set it: + +```toml +[path_defaults."**"] +description = "Order analytics model." + +[path_defaults.staging] +materialized = "view" +description = "Staging model." +``` ### Config layering order Configuration is layered in this order, with later layers overriding earlier ones: 1. **Project defaults** (`defaults`) -2. **Path defaults** (`path_defaults`) - if the model's path matches +2. **Path defaults** (`path_defaults`) - the one nearest matching entry, if any (see + [Which path default applies](#which-path-default-applies)) 3. **MODEL() header** - the model's own config Most keys are overridden by the more specific layer, but three merge instead: