diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index fe41979fd2..86e9f59e05 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -335,14 +335,61 @@ jobs: uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py -n auto --dist loadfile - - name: Compare python with the shipped native default on fixtures, examples, the failure corpus and a few seeds + - name: Compare python with native-preview on fixtures, examples, the failure corpus and a few seeds env: PYTHONUNBUFFERED: "1" run: >- uv run --no-sync python -m scripts.run_compiler_differential - --engines python native + --engines python native-preview --corpus fixtures examples failures seeds --seeds 4 --jobs 4 + native_discovery_windows: + name: Native discovery on Windows + needs: [changes, lockfile] + if: needs.changes.outputs.compiler_differential == 'true' + runs-on: windows-2025 + timeout-minutes: 40 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Test the native discovery crate (walk, junctions, ordering, read errors) + run: cargo test --locked -p sqlbuild-discovery + - name: Install Python + run: uv python install 3.12 + - name: Install SQLBuild and pytest + run: | + uv sync --locked --no-dev --python 3.12 + uv pip install "pytest>=8.4.2" "pytest-xdist>=3.8.0" + - name: Compare native text, YAML and TOML readers with this Python + run: >- + uv run --no-sync pytest + tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py + tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py + tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py + -n auto --dist loadfile + - name: Compile and build playground projects + run: | + project="${RUNNER_TEMP}/waffle-shop" + uv run --no-sync sqb playground "${project}" + uv run --no-sync sqb --project-dir "${project}" --no-color compile + SQLBUILD_COMPILER_ENGINE=python uv run --no-sync sqb --project-dir "${project}" --no-color compile + uv run --no-sync sqb --project-dir "${project}" --no-color build + nodes="${RUNNER_TEMP}/python-nodes" + uv run --no-sync sqb playground --template python_nodes "${nodes}" + uv run --no-sync sqb --project-dir "${nodes}" --no-color compile + format_performance: name: Formatter performance guards needs: [changes, lockfile] @@ -762,6 +809,7 @@ jobs: e2e_shards, compiler_differential, compiler_differential_python_versions, + native_discovery_windows, format_performance, performance, cold_compile_performance, @@ -783,6 +831,7 @@ jobs: E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }} + NATIVE_DISCOVERY_WINDOWS_RESULT: ${{ needs.native_discovery_windows.result }} FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }} PERFORMANCE_RESULT: ${{ needs.performance.result }} COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} @@ -806,6 +855,7 @@ jobs: "${E2E_SHARDS_RESULT}" \ "${COMPILER_DIFFERENTIAL_RESULT}" \ "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \ + "${NATIVE_DISCOVERY_WINDOWS_RESULT}" \ "${FORMAT_PERFORMANCE_RESULT}" \ "${PERFORMANCE_RESULT}" \ "${COLD_COMPILE_PERFORMANCE_RESULT}" \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed81fbfd22..d5cde6a4cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thanks for helping improve SQLBuild. Keep changes focused, tested, and easy to r ## Setup -SQLBuild requires Python 3.12 or newer. +SQLBuild supports Python 3.12, 3.13 and 3.14. Install dependencies with: diff --git a/Cargo.lock b/Cargo.lock index ab2ed994fd..ae317f66e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,6 +255,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "junction" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" +dependencies = [ + "scopeguard", + "windows-sys", +] + [[package]] name = "libc" version = "0.2.189" @@ -561,6 +571,12 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.229" @@ -664,6 +680,7 @@ version = "0.139.0" name = "sqlbuild-discovery" version = "0.139.0" dependencies = [ + "junction", "rayon", "sqlbuild-config", "sqlbuild-core", diff --git a/Cargo.toml b/Cargo.toml index 062b1353d6..4bdb6708f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,8 @@ rust-version = "1.88" [workspace.dependencies] fensu-policy = "=0.22.0" globset = "0.4" +# Windows only: tells NTFS junctions, which Python walks as directories, from symbolic links. +junction = { version = "2.1.0", default-features = false } # Python loads the extension with dlopen, so mimalloc must not use initial-exec TLS. mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } diff --git a/crates/README.md b/crates/README.md index 9b52dca374..8c6aad1340 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,8 +8,8 @@ that can be tested and benchmarked with `cargo test` alone. |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | -| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | -| `sqlbuild-discovery` | Native project discovery: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises; anything it cannot reproduce exactly defers to Python. | +| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | +| `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | | `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | diff --git a/crates/sqlbuild-config/src/errors.rs b/crates/sqlbuild-config/src/errors.rs index 972a4c07c9..53403b4652 100644 --- a/crates/sqlbuild-config/src/errors.rs +++ b/crates/sqlbuild-config/src/errors.rs @@ -15,7 +15,7 @@ pub enum ConfigErrorKind { InvalidField, } -/// A configuration failure; the Python reader re-parses the file to report its exact message. +/// A configuration failure, at its one-based line and column where known. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ConfigError { pub kind: ConfigErrorKind, @@ -34,4 +34,16 @@ impl ConfigError { column: None, } } + + /// This error at a one-based line and column, unless it already has a position. + pub fn at(self, line: usize, column: usize) -> Self { + if self.line.is_some() { + return self; + } + Self { + line: Some(line), + column: Some(column), + ..self + } + } } diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs index fee4e4b92e..130d68753f 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -64,10 +64,15 @@ fn syntax_error(error: &ScanError) -> ConfigError { kind: ConfigErrorKind::Syntax, message: error.info().to_owned(), line: Some(error.marker().line()), - column: Some(error.marker().col()), + column: Some(error.marker().col() + 1), } } +/// The one-based line and column where `span` starts. +fn start_position(span: Span) -> (usize, usize) { + (span.start.line(), span.start.col() + 1) +} + fn full_tag(tag: Option<&Cow<'_, Tag>>) -> Option { tag.map(|tag| format!("{}{}", tag.handle, tag.suffix)) } @@ -92,8 +97,12 @@ impl<'input> Composer<'input> { } } - fn push(&mut self, tag: String, content: NodeContent) -> usize { - self.document.nodes.push(Node { tag, content }); + fn push(&mut self, tag: String, content: NodeContent, span: Span) -> usize { + self.document.nodes.push(Node { + tag, + content, + position: start_position(span), + }); self.document.nodes.len() - 1 } @@ -103,7 +112,7 @@ impl<'input> Composer<'input> { return Ok(()); } let name = anchor_name(&self.chars, &self.code_ends, span.start.index()) - .ok_or_else(|| unsupported("anchor name could not be recovered"))?; + .ok_or_else(|| unsupported("this anchor"))?; if !is_python_anchor_name(&name) { return Err(ConfigError::new( ConfigErrorKind::Syntax, @@ -130,7 +139,7 @@ impl<'input> Composer<'input> { fn alias(&self, anchor_id: usize) -> Result { if self.open_anchors.contains(&anchor_id) { - return Err(unsupported("recursive aliases are left to Python")); + return Err(unsupported("recursive aliases")); } self.anchors .get(&anchor_id) @@ -212,7 +221,7 @@ impl<'input> Composer<'input> { && matches!(self.char_at(colon + 1), Some(',' | ']' | '}')); if placement.flow && adjacent_value { return Err(unsupported( - "LibYAML rejects ':' directly before a flow indicator", + "':' directly before ',', ']' or '}' in a flow collection", )); } let explicit_key = explicit_key_before(&self.chars, &self.code_ends, start); @@ -281,7 +290,7 @@ impl<'input> Composer<'input> { span: Span, placement: Placement, ) -> Result { - let deferred = || unsupported("this block scalar is left to Python"); + let deferred = || unsupported("this literal or folded block scalar layout"); if placement.flow || placement.key { return Err(deferred()); } @@ -304,14 +313,26 @@ impl<'input> Composer<'input> { .ok_or_else(deferred) } + /// Compose one node; an error without a position is placed at the node's start. fn compose_node( &mut self, event: Event<'input>, span: Span, placement: Placement, + ) -> Result { + let (line, column) = start_position(span); + self.compose_node_at(event, span, placement) + .map_err(|error| error.at(line, column)) + } + + fn compose_node_at( + &mut self, + event: Event<'input>, + span: Span, + placement: Placement, ) -> Result { if placement.depth > MAX_NESTING_DEPTH { - return Err(unsupported("nesting is deeper than the native limit")); + return Err(unsupported("nesting deeper than 256 levels")); } match event { Event::Alias(anchor_id) => self.alias(anchor_id), @@ -321,7 +342,7 @@ impl<'input> Composer<'input> { && value.contains(FLOW_KEY_INDICATOR) { return Err(unsupported( - "PyYAML ends a flow scalar at '?' and LibYAML does not", + "'?' inside a plain scalar in a flow collection", )); } if style == ScalarStyle::Plain @@ -340,9 +361,7 @@ impl<'input> Composer<'input> { )); } if placement.key && self.is_long_implicit_key(span) { - return Err(unsupported( - "implicit keys near PyYAML's 1024-character limit are left to Python", - )); + return Err(unsupported("implicit keys longer than 1000 characters")); } if placement.key && placement.flow && self.spans_lines(span) { return Err(ConfigError::new( @@ -361,13 +380,11 @@ impl<'input> Composer<'input> { }; let non_specific = full_tag(tag.as_ref()).as_deref() == Some(NON_SPECIFIC_TAG); if non_specific && value.is_empty() && style == ScalarStyle::Plain { - return Err(unsupported( - "LibYAML and PyYAML differ on an empty '!' scalar", - )); + return Err(unsupported("an empty scalar tagged '!'")); } self.open_anchor(anchor_id, span)?; let tag = scalar_tag(&value, style, full_tag(tag.as_ref())); - let node = self.push(tag, NodeContent::Scalar(value.into_owned())); + let node = self.push(tag, NodeContent::Scalar(value.into_owned()), span); Ok(self.close_anchor(anchor_id, node)) } Event::SequenceStart(anchor_id, tag) => { @@ -398,7 +415,7 @@ impl<'input> Composer<'input> { items.push(self.compose_node(event, item_span, item_placement)?); } let tag = collection_tag(full_tag(tag.as_ref()), SEQ_TAG); - let node = self.push(tag, NodeContent::Sequence(items)); + let node = self.push(tag, NodeContent::Sequence(items), span); Ok(self.close_anchor(anchor_id, node)) } @@ -426,7 +443,7 @@ impl<'input> Composer<'input> { entries.push((key, value)); } let tag = collection_tag(full_tag(tag.as_ref()), MAP_TAG); - let node = self.push(tag, NodeContent::Mapping(entries)); + let node = self.push(tag, NodeContent::Mapping(entries), span); Ok(self.close_anchor(anchor_id, node)) } diff --git a/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs index 10832a2862..a00eda6ec1 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs @@ -8,7 +8,7 @@ use crate::yaml::_helpers::timestamps::construct_timestamp; use crate::yaml::constants::{ BASE_EXPANSION_BUDGET, BOOL_TAG, BOOL_WORDS, EXPANSION_BUDGET_PER_CHARACTER, FLOAT_TAG, INT_TAG, MAP_TAG, MAX_CONSTRUCTION_DEPTH, MERGE_TAG, NULL_TAG, PYTHON_ONLY_TAGS, SEQ_TAG, - STR_TAG, TIMESTAMP_TAG, VALUE_TAG, + STANDARD_TAG_PREFIX, STR_TAG, TIMESTAMP_TAG, VALUE_TAG, }; use crate::yaml::models::{ComposedDocument, Node, NodeContent}; use std::collections::HashMap; @@ -17,6 +17,12 @@ use std::collections::hash_map::Entry; /// One mapping entry after merge keys are flattened; `string_key` marks a `=` key read as text. type FlatEntry = (usize, usize, bool); +/// A standard tag in its `!!name` shorthand. +fn short_tag(tag: &str) -> String { + tag.strip_prefix(STANDARD_TAG_PREFIX) + .map_or_else(|| tag.to_owned(), |name| format!("!!{name}")) +} + fn construct_error(message: String) -> ConfigError { ConfigError::new(ConfigErrorKind::Construct, message) } @@ -41,7 +47,7 @@ fn construct_scalar(tag: &str, text: &str) -> Result { TIMESTAMP_TAG => construct_timestamp(text.strip_suffix('\n').unwrap_or(text)), _ if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( ConfigErrorKind::Unsupported, - format!("values tagged {tag} are left to Python"), + format!("values tagged {}", short_tag(tag)), )), SEQ_TAG | MAP_TAG => Err(construct_error(format!( "expected a collection node for {tag}" @@ -114,24 +120,27 @@ impl Constructor<'_> { self.budget = self .budget .checked_sub(amount) - .ok_or_else(|| deferred("aliases expand beyond the native size budget"))?; + .ok_or_else(|| deferred("aliases that expand beyond the size limit"))?; Ok(()) } fn construct(&mut self, id: usize, depth: usize) -> Result { + let (line, column) = self.node(id).position; if depth > MAX_CONSTRUCTION_DEPTH { - return Err(deferred("values nest deeper than the native limit")); + return Err(deferred("values nested deeper than 256 levels").at(line, column)); } if let Some((value, size, height)) = self.built.get(&id) { if depth + height > MAX_CONSTRUCTION_DEPTH { - return Err(deferred("aliased values nest deeper than the native limit")); + return Err(deferred("values nested deeper than 256 levels").at(line, column)); } let (value, size) = (value.clone(), *size); - self.spend(size)?; + self.spend(size).map_err(|error| error.at(line, column))?; return Ok((value, size)); } - self.spend(1)?; - let built = self.construct_new(id, depth)?; + self.spend(1).map_err(|error| error.at(line, column))?; + let built = self + .construct_new(id, depth) + .map_err(|error| error.at(line, column))?; if self.shared[id] { let (value, size) = &built; self.built.insert(id, (value.clone(), *size, height(value))); @@ -157,7 +166,7 @@ impl Constructor<'_> { (NodeContent::Mapping(_), MAP_TAG) => self.construct_mapping(id, depth), (_, tag) if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( ConfigErrorKind::Unsupported, - format!("values tagged {tag} are left to Python"), + format!("values tagged {}", short_tag(tag)), )), (_, tag) => Err(construct_error(format!( "could not construct a collection tagged {tag:?}" @@ -200,7 +209,7 @@ impl Constructor<'_> { /// PyYAML's `flatten_mapping`: merged entries first, then the mapping's own entries. fn flatten(&mut self, id: usize, depth: usize) -> Result, ConfigError> { if depth > MAX_CONSTRUCTION_DEPTH { - return Err(deferred("merge keys nest deeper than the native limit")); + return Err(deferred("merge keys nested deeper than 256 levels")); } let mut merged: Vec = Vec::new(); let mut own: Vec = Vec::new(); diff --git a/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs index d756cfdcfa..f0f39c5513 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs @@ -130,7 +130,7 @@ pub(crate) fn construct_int(text: &str) -> Result { if value.len() > PYTHON_INT_MAX_STR_DIGITS { return Err(ConfigError::new( ConfigErrorKind::Unsupported, - "integers longer than Python's string conversion limit are left to Python", + "integers longer than 4300 digits", )); } let (inner_negative, magnitude) = if value == ZERO_TEXT { diff --git a/crates/sqlbuild-config/src/yaml/_helpers/reader.rs b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs index 9c568414be..0a0fac5686 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/reader.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs @@ -33,40 +33,88 @@ pub(crate) fn without_byte_order_mark(text: &str) -> &str { text.strip_prefix(BYTE_ORDER_MARK).unwrap_or(text) } -/// Reject what PyYAML's reader rejects, and defer characters PyYAML and LibYAML read differently. +/// The one-based line and column of the character at byte `index`. +fn position_of(text: &str, index: usize) -> (usize, usize) { + let before = &text[..index]; + let line_start = before.rfind('\n').map_or(0, |newline| newline + 1); + ( + before.matches('\n').count() + 1, + before[line_start..].chars().count() + 1, + ) +} + +fn error_at(text: &str, index: usize, kind: ConfigErrorKind, message: String) -> ConfigError { + let (line, column) = position_of(text, index); + ConfigError::new(kind, message).at(line, column) +} + +/// The byte index where the first line of `text` that `matches` starts. +fn line_index(text: &str, matches: impl Fn(&str) -> bool) -> Option { + let mut index = 0; + for line in text.split('\n') { + if matches(line) { + return Some(index); + } + index += line.len() + 1; + } + None +} + +/// Reject what PyYAML's reader rejects, and the characters PyYAML and LibYAML read differently. pub(crate) fn check_characters(text: &str) -> Result<(), ConfigError> { - if let Some(character) = text.chars().find(|character| !is_printable(*character)) { - return Err(ConfigError::new( + if let Some((index, character)) = text + .char_indices() + .find(|(_, character)| !is_printable(*character)) + { + return Err(error_at( + text, + index, ConfigErrorKind::Syntax, format!("unacceptable character {:#x}", u32::from(character)), )); } - if text.contains(PYTHON_ONLY_LINE_BREAKS) { - return Err(ConfigError::new( + if let Some(index) = text.find(PYTHON_ONLY_LINE_BREAKS) { + return Err(error_at( + text, + index, ConfigErrorKind::Unsupported, - "NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR are YAML 1.1 line breaks", + "NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR line breaks".to_owned(), )); } - if text.contains(LOADER_DEPENDENT_CHARACTERS) { - return Err(ConfigError::new( + if let Some(index) = text.find(LOADER_DEPENDENT_CHARACTERS) { + let construct: &str = if text[index..].starts_with('\t') { + "tab characters" + } else { + "byte order marks after the start of the document" + }; + return Err(error_at( + text, + index, ConfigErrorKind::Unsupported, - "tabs and inner byte order marks are read differently by PyYAML and LibYAML", + construct.to_owned(), )); } - let directive = text.starts_with(DIRECTIVE_INDICATOR) - || text - .split(['\n', '\r']) - .any(|line| line.starts_with(DIRECTIVE_INDICATOR)); - if directive { - return Err(ConfigError::new( + if let Some(index) = line_index(text, |line| { + line.starts_with(DIRECTIVE_INDICATOR) + || line + .split('\r') + .skip(1) + .any(|part| part.starts_with(DIRECTIVE_INDICATOR)) + }) { + return Err(error_at( + text, + index, ConfigErrorKind::Unsupported, - "directives are left to Python", + "%YAML and %TAG directives".to_owned(), )); } if starts_with_document_end(text) { - return Err(ConfigError::new( + let index = line_index(text, |line| !is_blank_or_comment(line)).unwrap_or(0); + return Err(error_at( + text, + index, ConfigErrorKind::Syntax, - "expected the node content, but found a document end marker", + "expected the node content, but found a document end marker".to_owned(), )); } Ok(()) diff --git a/crates/sqlbuild-config/src/yaml/constants.rs b/crates/sqlbuild-config/src/yaml/constants.rs index bd4f2ec34a..be4212f04f 100644 --- a/crates/sqlbuild-config/src/yaml/constants.rs +++ b/crates/sqlbuild-config/src/yaml/constants.rs @@ -1,5 +1,6 @@ //! YAML tags SafeLoader resolves and constructs. +pub(crate) const STANDARD_TAG_PREFIX: &str = "tag:yaml.org,2002:"; pub(crate) const NULL_TAG: &str = "tag:yaml.org,2002:null"; pub(crate) const BOOL_TAG: &str = "tag:yaml.org,2002:bool"; pub(crate) const INT_TAG: &str = "tag:yaml.org,2002:int"; diff --git a/crates/sqlbuild-config/src/yaml/models.rs b/crates/sqlbuild-config/src/yaml/models.rs index 01ade04ca4..36655fb837 100644 --- a/crates/sqlbuild-config/src/yaml/models.rs +++ b/crates/sqlbuild-config/src/yaml/models.rs @@ -13,6 +13,8 @@ pub(crate) enum NodeContent { pub(crate) struct Node { pub(crate) tag: String, pub(crate) content: NodeContent, + /// The one-based line and column where the node starts. + pub(crate) position: (usize, usize), } /// A composed document; `root` is `None` for an empty stream. diff --git a/crates/sqlbuild-core/src/text/errors.rs b/crates/sqlbuild-core/src/text/errors.rs index d69651f2cf..2f69b61688 100644 --- a/crates/sqlbuild-core/src/text/errors.rs +++ b/crates/sqlbuild-core/src/text/errors.rs @@ -4,4 +4,6 @@ #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TextDecodeError { pub valid_up_to: usize, + /// The length of the invalid sequence, or `None` when the bytes end mid-sequence. + pub error_len: Option, } diff --git a/crates/sqlbuild-core/src/text/main/decode_python_text.rs b/crates/sqlbuild-core/src/text/main/decode_python_text.rs index 0e32ea3c0f..86518491bd 100644 --- a/crates/sqlbuild-core/src/text/main/decode_python_text.rs +++ b/crates/sqlbuild-core/src/text/main/decode_python_text.rs @@ -6,6 +6,7 @@ use crate::text::errors::TextDecodeError; pub fn decode_python_text(bytes: &[u8]) -> Result { let text = std::str::from_utf8(bytes).map_err(|error| TextDecodeError { valid_up_to: error.valid_up_to(), + error_len: error.error_len(), })?; if !bytes.contains(&b'\r') { return Ok(text.to_owned()); diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index cb2d6a4aed..e5b3130fff 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -13,6 +13,9 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-sqltext.workspace = true +[target.'cfg(windows)'.dependencies] +junction.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-discovery/src/_helpers/reading.rs b/crates/sqlbuild-discovery/src/_helpers/reading.rs index 744d673df4..47b2848e01 100644 --- a/crates/sqlbuild-discovery/src/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/_helpers/reading.rs @@ -1,11 +1,135 @@ //! Read authored files as Python's `Path.read_text(encoding="utf-8")` does. -use crate::models::ReadFailure; +use crate::models::{DiscoveryFailure, FailureKind, ProjectRoot, ReadFailure}; +use crate::tree::models::ProjectTree; +use sqlbuild_core::text::errors::TextDecodeError; use sqlbuild_core::text::main::decode_python_text::decode_python_text; use std::path::Path; -/// The decoded text, or why Python's read would raise. +/// The high half of a `FACILITY_WIN32` HRESULT, which wraps a Win32 error in its low half. +const WIN32_HRESULT_HIGH: u32 = 0x8007_0000; +const HRESULT_HIGH_MASK: u32 = 0xFFFF_0000; +const HRESULT_CODE_MASK: i32 = 0xFFFF; + +/// The decoded text, or the error Python's read raises. pub(crate) fn read_authored_text(path: &Path) -> Result { - let bytes: Vec = std::fs::read(path).map_err(|error| ReadFailure::Io(error.kind()))?; - decode_python_text(&bytes).map_err(ReadFailure::Decode) + let bytes: Vec = std::fs::read(path).map_err(|error| read_failure(&error))?; + decode_python_text(&bytes).map_err(|error| decode_failure(bytes, error)) +} + +/// `D016` for a file discovery reads whose path holds a name that is not valid UTF-8. +pub(crate) fn undecodable_path_failure( + root: &ProjectRoot, + tree: &ProjectTree, + relative_path: &str, +) -> Option { + tree.is_undecodable(relative_path).then(|| { + DiscoveryFailure::new( + FailureKind::ProjectPath, + format!( + "Project path {} is not valid UTF-8; rename it so SQLBuild can read it", + root.display_path(relative_path) + ), + ) + }) +} + +/// Python's `OSError` for a refused file read on this platform. +pub(crate) fn read_failure(error: &std::io::Error) -> ReadFailure { + os_read_failure(error.raw_os_error(), error.to_string(), cfg!(windows)) +} + +/// Python's `OSError` for a directory that cannot be listed on this platform. +pub(crate) fn listing_failure(error: &std::io::Error) -> ReadFailure { + os_listing_failure(error.raw_os_error(), error.to_string(), cfg!(windows)) +} + +/// `open()` reports the C runtime's errno; on Windows the C runtime maps the Win32 error. +pub(crate) fn os_read_failure(code: Option, message: String, windows: bool) -> ReadFailure { + ReadFailure::Io { + errno: if windows { + code.map(windows_errno) + } else { + code + }, + winerror: None, + message, + } +} + +/// Listing reports the POSIX errno, or on Windows the Win32 error and its system message. +pub(crate) fn os_listing_failure(code: Option, message: String, windows: bool) -> ReadFailure { + if windows { + ReadFailure::Io { + errno: None, + winerror: code, + message: windows_message(&message), + } + } else { + ReadFailure::Io { + errno: code, + winerror: None, + message, + } + } +} + +/// The system message without Rust's ` (os error N)` and the trailing dots CPython strips. +fn windows_message(message: &str) -> String { + let text: &str = message + .rfind(" (os error ") + .map_or(message, |end| &message[..end]); + text.trim_end_matches(|character: char| character == '.' || character <= ' ') + .to_owned() +} + +/// CPython's `winerror_to_errno` (`PC/errmap.h`), which mirrors the C runtime's mapping. +pub(crate) fn windows_errno(code: i32) -> i32 { + let code: i32 = if code.cast_unsigned() & HRESULT_HIGH_MASK == WIN32_HRESULT_HIGH { + code & HRESULT_CODE_MASK + } else { + code + }; + match code { + 10004 | 10009 | 10013 | 10014 | 10022 | 10024 => code - 10000, + 10000..=11999 => code, + 2 | 3 | 15 | 18 | 53 | 67 | 161 | 206 => 2, + 10 => 7, + 11 | 188..=202 => 8, + 6 | 114 | 130 => 9, + 128 | 129 => 10, + 89 | 164 | 215 => 11, + 7 | 8 | 9 | 1816 => 12, + 5 | 16 | 19..=36 | 65 | 82 | 83 | 108 | 132 | 158 | 167 => 13, + 80 | 183 => 17, + 17 => 18, + 267 => 20, + 4 => 24, + 112 => 28, + 109 | 232 => 32, + 145 => 41, + 1113 => 42, + _ => 22, + } +} + +/// CPython's UTF-8 decoder error: the range and reason it reports for the first bad sequence. +fn decode_failure(bytes: Vec, error: TextDecodeError) -> ReadFailure { + let start: usize = error.valid_up_to; + let (end, reason): (usize, &'static str) = match error.error_len { + None => (bytes.len(), "unexpected end of data"), + Some(length) if !is_lead_byte(bytes[start]) => (start + length, "invalid start byte"), + Some(length) => (start + length, "invalid continuation byte"), + }; + ReadFailure::Decode { + bytes, + start, + end, + reason, + } +} + +/// Whether a byte may start a UTF-8 sequence (ASCII never fails to decode). +fn is_lead_byte(byte: u8) -> bool { + (0xC2..=0xF4).contains(&byte) } diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 32730b15f6..1c58b4febe 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -39,8 +39,6 @@ pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [ ]; /// Project-wide named declaration roles. pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"]; -/// Root of seed files, walked so the Python snapshot shares its listings. -pub const SEEDS_DIRECTORY: &str = "seeds"; /// Suffix of macro files. pub const PYTHON_FILE_SUFFIX: &str = ".py"; /// The stem of package initialisers, which are not macro files. diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs index 7172da607a..0263ca2441 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_groups.rs @@ -1,4 +1,4 @@ -//! Python's `_validate_declaration_groups` and `_declaration_groups`. +//! Validate scoped `_sqlbuild` declaration groups and list the groups below each owner. use crate::constants::{ CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GROUPED_NAMED_DECLARATION_DIRECTORIES, @@ -8,13 +8,13 @@ use crate::declarations::_helpers::paths::parent; use crate::declarations::_helpers::scan::declaration_failure; use crate::declarations::errors::ScanError; use crate::declarations::models::DeclarationGroup; -use crate::models::StageDeferral; +use crate::models::StageFailure; use crate::tree::main::children::children; use crate::tree::main::rglob::rglob; use crate::tree::models::{ProjectTree, TreeEntry}; /// Every `_sqlbuild/` directory below a canonical root, root by root, in path order. -fn all_groups(tree: &ProjectTree) -> Result, StageDeferral> { +fn all_groups(tree: &ProjectTree) -> Result, StageFailure> { let mut groups: Vec = Vec::new(); for root_parts in CANONICAL_AUTHORED_ROOTS { let root: String = root_parts.join("/"); @@ -35,8 +35,8 @@ fn all_groups(tree: &ProjectTree) -> Result, StageDeferral Ok(groups) } -/// Python's `_declaration_groups`: the groups below a concrete owner directory. -pub(crate) fn owned_groups(tree: &ProjectTree) -> Result, StageDeferral> { +/// The declaration groups below a concrete owner directory. +pub(crate) fn owned_groups(tree: &ProjectTree) -> Result, StageFailure> { Ok(all_groups(tree)? .into_iter() .filter(|group| parent(&group.directory) != group.root) diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs index 8f99a20d08..ee4440329a 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -1,4 +1,4 @@ -//! Python's `_scan_declaration_file_facts`: macro, enum and constant files with scope facts. +//! Scan macro, enum and constant declaration files with their scope facts. use crate::constants::{ CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, @@ -11,6 +11,7 @@ use crate::declarations::_helpers::scan::declaration_failure; use crate::declarations::errors::ScanError; use crate::declarations::models::{DeclarationFileFact, DeclarationKind, ScopeKind}; use crate::tree::main::directories::directories; +use crate::tree::main::path_order::compare_posix_text; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; @@ -25,11 +26,15 @@ struct DeclarationRoot<'a> { /// Every declaration file below every macro, enum and constant root, by relative path. pub(crate) fn declaration_file_facts( tree: &ProjectTree, + kind: Option, ) -> Result, ScanError> { validate_declaration_groups(tree)?; let mut local: Vec<&str> = LOCAL_DECLARATION_DIRECTORIES.to_vec(); local.sort_unstable(); - if let Some(directory) = local.into_iter().find(|directory| tree.is_dir(directory)) { + if let Some(directory) = local + .into_iter() + .find(|directory| is_kind(kind, directory) && tree.is_dir(directory)) + { return Err(declaration_failure(format!( "Scoped declaration root {directory}/ must be below a canonical authored root" ))); @@ -38,7 +43,7 @@ pub(crate) fn declaration_file_facts( let mut global: Vec<&str> = GLOBAL_DECLARATION_DIRECTORIES.to_vec(); global.sort_unstable(); for directory in global { - if tree.is_dir(directory) { + if is_kind(kind, directory) && tree.is_dir(directory) { facts.extend(files_under_root( tree, &DeclarationRoot { @@ -53,10 +58,10 @@ pub(crate) fn declaration_file_facts( for root_parts in CANONICAL_AUTHORED_ROOTS { let root: String = root_parts.join("/"); if tree.is_dir(&root) { - facts.extend(scoped_root_files(tree, &root, root_parts.len())?); + facts.extend(scoped_root_files(tree, &root, root_parts.len(), kind)?); } } - facts.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + facts.sort_by(|left, right| compare_posix_text(&left.relative_path, &right.relative_path)); Ok(facts) } @@ -64,12 +69,16 @@ fn scoped_root_files( tree: &ProjectTree, root: &str, root_length: usize, + kind: Option, ) -> Result, ScanError> { let mut facts: Vec = Vec::new(); for directory in directories(tree, root)? { let Some((_, scope_kind)) = directory_facts(name(&directory)) else { continue; }; + if !is_kind(kind, name(&directory)) { + continue; + } let parts: Vec<&str> = directory.split('/').collect(); let between: &[&str] = &parts[root_length..parts.len() - 1]; if between @@ -95,6 +104,15 @@ fn scoped_root_files( Ok(facts) } +/// Whether a declaration directory holds `kind`, or every kind when none is requested. +fn is_kind(kind: Option, directory_name: &str) -> bool { + match (kind, directory_facts(directory_name)) { + (None, _) => true, + (Some(kind), Some((directory_kind, _))) => directory_kind == kind, + (Some(_), None) => false, + } +} + fn nested_root_failure(directory: &str) -> ScanError { declaration_failure(format!( "Declaration root {directory}/ is nested inside another declaration tree" diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs index 073d9d6dd8..0270b2fd10 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/named_layout.rs @@ -1,4 +1,4 @@ -//! Python's `validate_named_declaration_layout`. +//! Validate where named declaration roots may appear in the project layout. use crate::constants::{ AUDIT_ROLE_DIRECTORY, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs index ce4fce93d0..688b4e0601 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/scan.rs @@ -1,20 +1,20 @@ //! Building and splitting declaration scan errors. use crate::declarations::errors::ScanError; -use crate::models::{DiscoveryFailure, FailureKind, StageDeferral}; +use crate::models::{DiscoveryFailure, FailureKind, StageFailure}; /// A `DeclarationParseError` with Python's message. pub(crate) fn declaration_failure(message: String) -> ScanError { ScanError::Failure(DiscoveryFailure::new(FailureKind::Declaration, message)) } -/// Split a scan result into the authored outcome, keeping a deferral for the caller. +/// Split a scan result into the authored outcome, keeping a stage failure for the caller. pub(crate) fn authored_outcome( result: Result, -) -> Result, StageDeferral> { +) -> Result, StageFailure> { match result { Ok(value) => Ok(Ok(value)), Err(ScanError::Failure(failure)) => Ok(Err(failure)), - Err(ScanError::Deferral(deferral)) => Err(deferral), + Err(ScanError::Stage(failure)) => Err(failure), } } diff --git a/crates/sqlbuild-discovery/src/declarations/errors.rs b/crates/sqlbuild-discovery/src/declarations/errors.rs index 88ae69bf1b..1585e5100c 100644 --- a/crates/sqlbuild-discovery/src/declarations/errors.rs +++ b/crates/sqlbuild-discovery/src/declarations/errors.rs @@ -1,14 +1,14 @@ -//! Why a declaration scan stopped: an authored layout failure or a whole-stage deferral. +//! Why a declaration scan stopped: an authored layout failure or a whole-stage failure. -use crate::models::{DiscoveryFailure, StageDeferral}; +use crate::models::{DiscoveryFailure, StageFailure}; pub(crate) enum ScanError { Failure(DiscoveryFailure), - Deferral(StageDeferral), + Stage(StageFailure), } -impl From for ScanError { - fn from(deferral: StageDeferral) -> Self { - Self::Deferral(deferral) +impl From for ScanError { + fn from(failure: StageFailure) -> Self { + Self::Stage(failure) } } diff --git a/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs index 6b775310ad..80730c01bd 100644 --- a/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs +++ b/crates/sqlbuild-discovery/src/declarations/main/declaration_layout.rs @@ -1,22 +1,22 @@ //! Walk and validate the declaration layout the Python declaration collections read. use crate::_helpers::pool::discovery_pool; -use crate::constants::SEEDS_DIRECTORY; use crate::declarations::_helpers::declaration_tree::declaration_file_facts; use crate::declarations::_helpers::named_layout::named_declaration_groups; use crate::declarations::_helpers::scan::authored_outcome; -use crate::declarations::models::DeclarationLayout; -use crate::models::StageDeferral; -use crate::tree::main::read_tree::read_tree; +use crate::declarations::models::{DeclarationKind, DeclarationLayout}; +use crate::models::StageFailure; use crate::tree::models::ProjectTree; -/// The declaration file facts and named declaration groups, each valid or failing as in Python. -pub fn declaration_layout(tree: &ProjectTree) -> Result { - let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; +/// The file facts (of `kind` only, if given) and named declaration groups, valid or failing. +pub fn declaration_layout( + tree: &ProjectTree, + kind: Option, +) -> Result { + let pool = discovery_pool().map_err(StageFailure::Internal)?; pool.install(|| { - let file_facts = authored_outcome(declaration_file_facts(tree))?; + let file_facts = authored_outcome(declaration_file_facts(tree, kind))?; let named_groups = authored_outcome(named_declaration_groups(tree))?; - read_tree(tree, SEEDS_DIRECTORY)?; Ok(DeclarationLayout { file_facts, named_groups, diff --git a/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs index e39cfcf614..dbbe9f3887 100644 --- a/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/declarations/tests/helpers.rs @@ -21,7 +21,8 @@ pub(super) fn layout_rows( fs::create_dir_all(project.path().join(relative_path)).expect("directory"); } let layout: DeclarationLayout = - declaration_layout(&ProjectTree::new(project.path())).expect("no deferral"); + declaration_layout(&ProjectTree::new(project.path()), test_case.kind) + .expect("no stage failure"); ( rows(layout.file_facts, fact_row), rows(layout.named_groups, group_pair), diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs index 1cab16a64b..e1c30022dc 100644 --- a/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_declaration_layout.rs @@ -1,3 +1,4 @@ +use crate::declarations::models::DeclarationKind; use crate::declarations::tests::helpers::{layout_rows, owned_facts, owned_groups}; use crate::declarations::tests::test_types::LayoutTestCase; @@ -5,6 +6,7 @@ use crate::declarations::tests::test_types::LayoutTestCase; fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python() { let test_cases = [ LayoutTestCase { + kind: None, description: "global, inherited, grouped and local roots sort by relative path", files: &[ "macros/text.py", @@ -45,6 +47,7 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( expected_groups: Ok(&[("models", "models/marts/_sqlbuild")]), }, LayoutTestCase { + kind: None, description: "a declaration group directly below a canonical root fails", files: &["models/_sqlbuild/macros/text.py"], directories: &[], @@ -56,6 +59,7 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( expected_groups: Ok(&[]), }, LayoutTestCase { + kind: None, description: "a nested declaration root fails in both scans", files: &["models/a/_enums/macros/x.py"], directories: &["schemas/enums"], @@ -70,6 +74,7 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( ), }, LayoutTestCase { + kind: None, description: "unsupported group and role entries list every entry", files: &[ "models/a/_sqlbuild/notes.txt", @@ -87,6 +92,7 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( ), }, LayoutTestCase { + kind: None, description: "local singular audits and unknown hook roles fail the named layout", files: &[], directories: &[ @@ -101,6 +107,7 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( ), }, LayoutTestCase { + kind: None, description: "a scoped root at the project root fails", files: &["_macros/text.py"], directories: &[], @@ -109,6 +116,29 @@ fn given_declaration_layouts_when_scanning_then_facts_and_failures_match_python( ), expected_groups: Ok(&[]), }, + LayoutTestCase { + kind: Some(DeclarationKind::Enum), + description: "an isolated kind skips the misplaced roots of other kinds", + files: &[ + "_macros/text.py", + "enums/status.sql", + "models/a/macros/x.py", + ], + directories: &[], + expected_facts: Ok(&[("enums/status.sql", "enum", "global", "enums", None, "enums")]), + expected_groups: Ok(&[]), + }, + LayoutTestCase { + kind: Some(DeclarationKind::Constant), + description: "an isolated kind still rejects a root nested in its own tree", + files: &["models/a/constants/_enums/x.sql"], + directories: &[], + expected_facts: Err( + "Declaration root models/a/constants/_enums/ is nested inside another \ + declaration tree", + ), + expected_groups: Ok(&[]), + }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs index 458cbc4004..d4170d6198 100644 --- a/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/declarations/tests/test_types.rs @@ -1,5 +1,9 @@ +use crate::declarations::models::DeclarationKind; + pub(super) struct LayoutTestCase { pub(super) description: &'static str, + /// The only declaration kind scanned, or every kind. + pub(super) kind: Option, pub(super) files: &'static [&'static str], pub(super) directories: &'static [&'static str], /// `(relative path, kind, scope kind, ownership root, owning path, declaration root)`. diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index 24c0a3a454..accf0a32ca 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -1,4 +1,4 @@ -//! Parse one model file with the checks and messages of Python's `parse_matched_model_sql`. +//! Parse one model file: header checks, model-local declaration checks and their messages. use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::model_files::_helpers::locations::header_column_locations; diff --git a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs index 0255001251..a365eddd22 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/discover_model_files.rs @@ -1,12 +1,12 @@ //! Discover, read and parse the SQL model files under `models/`. use crate::_helpers::pool::discovery_pool; -use crate::_helpers::reading::read_authored_text; +use crate::_helpers::reading::{read_authored_text, undecodable_path_failure}; use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; use crate::constants::{MODELS_DIRECTORY, SQL_FILE_SUFFIX}; use crate::model_files::_helpers::parsing::parse_model_file; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; -use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageDeferral}; +use crate::models::{DiscoveredFile, FileOutcome, ProjectRoot, StageFailure}; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -16,8 +16,8 @@ pub fn discover_model_files( root: &ProjectRoot, tree: &ProjectTree, options: &ModelFileOptions, -) -> Result>, StageDeferral> { - let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; +) -> Result>, StageFailure> { + let pool = discovery_pool().map_err(StageFailure::Internal)?; pool.install(|| { let paths: Vec = rglob(tree, MODELS_DIRECTORY, |entry| { entry.name.ends_with(SQL_FILE_SUFFIX) @@ -39,8 +39,12 @@ fn discover_one( relative_path: &str, options: &ModelFileOptions, ) -> FileOutcome { - let Ok(contents) = read_authored_text(&tree.absolute(relative_path)) else { - return FileOutcome::Unreadable; + if let Some(failure) = undecodable_path_failure(root, tree, relative_path) { + return FileOutcome::Failed(failure); + } + let contents: String = match read_authored_text(&tree.absolute(relative_path)) { + Ok(contents) => contents, + Err(failure) => return FileOutcome::Unreadable(failure), }; match parse_model_file(&root.display_path(relative_path), contents, options) { Ok(model_file) => FileOutcome::Parsed(model_file), diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index 4d9ade76d4..278eb30e86 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1 +1,2 @@ pub mod discover_model_files; +pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/main/parse_model_text.rs b/crates/sqlbuild-discovery/src/model_files/main/parse_model_text.rs new file mode 100644 index 0000000000..4084878b35 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/parse_model_text.rs @@ -0,0 +1,14 @@ +//! Parse SQL model contents that are already in memory. + +use crate::model_files::_helpers::parsing::parse_model_file; +use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use crate::models::DiscoveryFailure; + +/// One model file's parsed contents, named `file_path` in messages. +pub fn parse_model_text( + file_path: &str, + contents: String, + options: &ModelFileOptions, +) -> Result { + parse_model_file(file_path, contents, options) +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index 9b0680bd90..2ea146bb9a 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::model_files::_helpers::parsing::parse_model_file; +use crate::model_files::main::parse_model_text::parse_model_text; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; use crate::models::LineColumnSpan; @@ -50,7 +50,7 @@ fn summary(model: &DiscoveredModelFile) -> (String, Vec, Vec) /// Parse one model file as `/project/models/orders.sql` and summarise the outcome. pub(super) fn parsed_summary(contents: &str) -> ModelSummary { - parse_model_file(FILE_PATH, contents.to_owned(), &options()) + parse_model_text(FILE_PATH, contents.to_owned(), &options()) .map(|model| summary(&model)) .map_err(|failure| (failure.message, failure.help)) } diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 6e2ff16226..7918049945 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -1,6 +1,6 @@ //! Results native discovery hands to the Python facade. -use sqlbuild_core::text::errors::TextDecodeError; +use crate::tree::main::display_text::display_text; use std::path::PathBuf; /// The Python `DiscoveryError` subclass a native failure is raised as. @@ -14,6 +14,12 @@ pub enum FailureKind { SqlTest, /// `SqlScenarioParseError` (D009). SqlScenario, + /// `SchemaParseError` (D005). + Schema, + /// `SourceParseError` (D006). + Source, + /// `ProjectPathError` (D016). + ProjectPath, } impl FailureKind { @@ -24,6 +30,9 @@ impl FailureKind { Self::Declaration => "declaration", Self::SqlTest => "sql_test", Self::SqlScenario => "sql_scenario", + Self::Schema => "schema", + Self::Source => "source", + Self::ProjectPath => "project_path", } } } @@ -47,10 +56,16 @@ impl DiscoveryFailure { } } -/// Native discovery cannot reproduce Python for this project; the Python stage runs instead. +/// Why a whole discovery collection cannot be read. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct StageDeferral { - pub reason: String, +pub enum StageFailure { + /// A directory Python lists with `iterdir()` could not be listed: Python's `OSError`. + Unlistable { + relative_path: String, + error: ReadFailure, + }, + /// The discovery worker pool could not start. + Internal(String), } /// The outcome of reading and parsing one authored file. @@ -58,8 +73,8 @@ pub struct StageDeferral { pub enum FileOutcome { Parsed(T), Failed(DiscoveryFailure), - /// The bytes could not be read or decoded; Python re-reads the file to raise its error. - Unreadable, + /// The file could not be read or decoded as Python's `read_text` would. + Unreadable(ReadFailure), } /// One discovered file, by `/`-separated path relative to the project directory. @@ -88,13 +103,28 @@ pub struct ProjectRoot { impl ProjectRoot { /// Python's `str(project_dir / relative_path)` for a `/`-separated relative path. pub fn display_path(&self, relative_path: &str) -> String { - format!("{}{relative_path}", self.display_prefix) + format!( + "{}{}", + self.display_prefix, + display_text(relative_path).replace('/', std::path::MAIN_SEPARATOR_STR) + ) } } /// Why an authored file could not be read the way Python's `read_text` reads it. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum ReadFailure { - Io(std::io::ErrorKind), - Decode(TextDecodeError), + /// A refused read: `OSError(errno, strerror, path)`, or `winerror` for a Windows listing. + Io { + errno: Option, + winerror: Option, + message: String, + }, + /// Python's `UnicodeDecodeError("utf-8", bytes, start, end, reason)`. + Decode { + bytes: Vec, + start: usize, + end: usize, + reason: &'static str, + }, } diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs index 3cb6794612..f16696eaf6 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/scenario_parsing.rs @@ -1,4 +1,4 @@ -//! The header half of Python's `parse_sql_scenario_file`. +//! Parse a scenario file header; Python builds the scenario from the parsed values. use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::models::{DiscoveryFailure, FailureKind}; diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs index 6641b2950c..caf1f77e30 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/selection.rs @@ -1,10 +1,10 @@ //! Read and parse the unscoped SQL files below one root in parallel, in Python's order. use crate::_helpers::pool::discovery_pool; -use crate::_helpers::reading::read_authored_text; +use crate::_helpers::reading::{read_authored_text, undecodable_path_failure}; use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; use crate::constants::SQL_FILE_SUFFIX; -use crate::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, ProjectRoot, StageDeferral}; +use crate::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, ProjectRoot, StageFailure}; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -15,8 +15,8 @@ pub(crate) fn discover_sql_files( tree: &ProjectTree, directory: &str, parse: impl Fn(&str, String) -> Result + Sync, -) -> Result>, StageDeferral> { - let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; +) -> Result>, StageFailure> { + let pool = discovery_pool().map_err(StageFailure::Internal)?; pool.install(|| { let paths: Vec = rglob(tree, directory, |entry| { entry.name.ends_with(SQL_FILE_SUFFIX) @@ -25,15 +25,27 @@ pub(crate) fn discover_sql_files( .into_par_iter() .filter(|relative_path| !is_in_scoped_declaration_tree(relative_path)) .map(|relative_path| DiscoveredFile { - outcome: match read_authored_text(&tree.absolute(&relative_path)) { - Ok(contents) => match parse(&root.display_path(&relative_path), contents) { - Ok(parsed) => FileOutcome::Parsed(parsed), - Err(failure) => FileOutcome::Failed(failure), - }, - Err(_unreadable) => FileOutcome::Unreadable, - }, + outcome: read_and_parse(root, tree, &relative_path, &parse), relative_path, }) .collect()) }) } + +fn read_and_parse( + root: &ProjectRoot, + tree: &ProjectTree, + relative_path: &str, + parse: &impl Fn(&str, String) -> Result, +) -> FileOutcome { + if let Some(failure) = undecodable_path_failure(root, tree, relative_path) { + return FileOutcome::Failed(failure); + } + match read_authored_text(&tree.absolute(relative_path)) { + Ok(contents) => match parse(&root.display_path(relative_path), contents) { + Ok(parsed) => FileOutcome::Parsed(parsed), + Err(failure) => FileOutcome::Failed(failure), + }, + Err(failure) => FileOutcome::Unreadable(failure), + } +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs index 4c5541f6c0..34c47e22fa 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/_helpers/test_blocks.rs @@ -1,4 +1,4 @@ -//! Python's `_split_sql_test_blocks` and the header half of `_parse_single_sql_test_block`. +//! Split a SQL test file into `TEST` blocks and parse each block header. use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::models::{DiscoveryFailure, FailureKind}; diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs index 3bf3d87fb4..ccbf905907 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_scenario_files.rs @@ -1,7 +1,7 @@ //! Discover, read and header-parse the SQL scenario files under `tests/scenarios/`. use crate::constants::SQL_SCENARIOS_ROOT; -use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::models::{DiscoveredFile, ProjectRoot, StageFailure}; use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; use crate::sql_tests::_helpers::selection::discover_sql_files; use crate::sql_tests::models::{DiscoveredScenarioFile, SqlTestFileOptions}; @@ -12,7 +12,7 @@ pub fn discover_scenario_files( root: &ProjectRoot, tree: &ProjectTree, options: &SqlTestFileOptions, -) -> Result>, StageDeferral> { +) -> Result>, StageFailure> { discover_sql_files(root, tree, SQL_SCENARIOS_ROOT, |file_path, contents| { parse_scenario_file(file_path, contents, options) }) diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs index 27aded3510..908475a3a0 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/discover_sql_test_files.rs @@ -1,7 +1,7 @@ //! Discover, read and split the SQL unit-test files under `tests/unit/`. use crate::constants::SQL_TESTS_ROOT; -use crate::models::{DiscoveredFile, ProjectRoot, StageDeferral}; +use crate::models::{DiscoveredFile, ProjectRoot, StageFailure}; use crate::sql_tests::_helpers::selection::discover_sql_files; use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; @@ -12,7 +12,7 @@ pub fn discover_sql_test_files( root: &ProjectRoot, tree: &ProjectTree, options: &SqlTestFileOptions, -) -> Result>, StageDeferral> { +) -> Result>, StageFailure> { discover_sql_files( root, tree, diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs index f79f30314c..032628c3f1 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/main/mod.rs @@ -1,2 +1,4 @@ pub mod discover_scenario_files; pub mod discover_sql_test_files; +pub mod parse_scenario_text; +pub mod parse_sql_test_text; diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/parse_scenario_text.rs b/crates/sqlbuild-discovery/src/sql_tests/main/parse_scenario_text.rs new file mode 100644 index 0000000000..e766ca35ff --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/parse_scenario_text.rs @@ -0,0 +1,14 @@ +//! Header-parse SQL scenario contents that are already in memory. + +use crate::models::DiscoveryFailure; +use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; +use crate::sql_tests::models::{DiscoveredScenarioFile, SqlTestFileOptions}; + +/// The parsed header and body of one scenario file's contents, named `file_path` in messages. +pub fn parse_scenario_text( + file_path: &str, + contents: String, + options: &SqlTestFileOptions, +) -> Result { + parse_scenario_file(file_path, contents, options) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/main/parse_sql_test_text.rs b/crates/sqlbuild-discovery/src/sql_tests/main/parse_sql_test_text.rs new file mode 100644 index 0000000000..3010f15752 --- /dev/null +++ b/crates/sqlbuild-discovery/src/sql_tests/main/parse_sql_test_text.rs @@ -0,0 +1,14 @@ +//! Split SQL unit-test contents that are already in memory. + +use crate::models::DiscoveryFailure; +use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; + +/// The header-parsed blocks of one SQL test file's contents, named `file_path` in messages. +pub fn parse_sql_test_text( + file_path: &str, + contents: String, + options: &SqlTestFileOptions, +) -> Result { + parse_sql_test_file(file_path, contents, options) +} diff --git a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs index 9713220e26..3abaa38ac5 100644 --- a/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/sql_tests/tests/helpers.rs @@ -1,6 +1,6 @@ use crate::models::DiscoveryFailure; -use crate::sql_tests::_helpers::scenario_parsing::parse_scenario_file; -use crate::sql_tests::_helpers::test_blocks::parse_sql_test_file; +use crate::sql_tests::main::parse_scenario_text::parse_scenario_text; +use crate::sql_tests::main::parse_sql_test_text::parse_sql_test_text; use crate::sql_tests::models::{SqlTestBlock, SqlTestFileOptions}; use crate::sql_tests::tests::test_types::StatementFileTestCase; use sqlbuild_core::text::main::python_text::python_text; @@ -41,7 +41,7 @@ fn failed_rows(failure: DiscoveryFailure) -> (Vec, Option) { /// The test file's parsed block rows and its failure message. pub(super) fn test_file_rows(test_case: &StatementFileTestCase) -> (Vec, Option) { - parse_sql_test_file(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( + parse_sql_test_text(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( failed_rows, |file| { ( @@ -54,7 +54,7 @@ pub(super) fn test_file_rows(test_case: &StatementFileTestCase) -> (Vec (Vec, Option) { - parse_scenario_file(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( + parse_scenario_text(FILE_PATH, test_case.contents.to_owned(), &options()).map_or_else( failed_rows, |file| { ( diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs index 5946236045..51fa17cb8f 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs @@ -1,42 +1,58 @@ //! One directory listing with the entry type facts Python's `os.scandir` reports. -use crate::models::StageDeferral; +use crate::tree::_helpers::raw_names::raw_segment; use crate::tree::models::TreeEntry; -use std::fs::DirEntry; +use std::fs::{DirEntry, FileType}; use std::path::Path; /// List a directory; an unreadable directory lists as empty, as in Python's snapshot. -pub(crate) fn list_directory(directory: &Path) -> Result, StageDeferral> { +pub(crate) fn list_directory(directory: &Path) -> Vec { let Ok(entries) = std::fs::read_dir(directory) else { - return Ok(Vec::new()); + return Vec::new(); }; let mut listing: Vec = Vec::new(); for entry in entries { let Ok(entry) = entry else { - return Ok(Vec::new()); + return Vec::new(); }; - listing.push(tree_entry(&entry)?); + listing.push(tree_entry(&entry)); } - Ok(listing) + listing } -fn tree_entry(entry: &DirEntry) -> Result { - let name = entry - .file_name() - .into_string() - .map_err(|name| StageDeferral { - reason: format!("file name is not UTF-8: {}", name.to_string_lossy()), - })?; +fn tree_entry(entry: &DirEntry) -> TreeEntry { + let (name, segment, raw_name) = match entry.file_name().into_string() { + Ok(name) => (name.clone(), name, None), + Err(raw) => { + let lossy: String = raw.to_string_lossy().into_owned(); + let segment: String = raw_segment(&lossy, &raw); + (lossy, segment, Some(raw)) + } + }; let (is_dir, is_walkable_dir) = match entry.file_type() { + Ok(file_type) if is_junction(entry, &file_type) => (true, true), Ok(file_type) if file_type.is_symlink() => (link_target_is_dir(entry), false), Ok(file_type) => (file_type.is_dir(), file_type.is_dir()), Err(_unreadable) => (false, false), }; - Ok(TreeEntry { + TreeEntry { name, + raw_name, + segment, is_dir, is_walkable_dir, - }) + } +} + +/// An NTFS junction, which Python's `os.scandir` reports as a real directory and walks into. +#[cfg(windows)] +fn is_junction(entry: &DirEntry, file_type: &FileType) -> bool { + file_type.is_symlink() && matches!(junction::exists(entry.path()), Ok(true)) +} + +#[cfg(not(windows))] +fn is_junction(_entry: &DirEntry, _file_type: &FileType) -> bool { + false } fn link_target_is_dir(entry: &DirEntry) -> bool { diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs b/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs index c484bf675a..4c31fbe9bd 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod listing; pub(crate) mod ordering; +pub(crate) mod raw_names; pub(crate) mod walking; diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs b/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs index defef4fcc7..bfaf06e104 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/ordering.rs @@ -1,7 +1,39 @@ //! Python's `sorted()` order of paths: part by part, each part by code point. +use crate::tree::_helpers::raw_names::{ + has_raw_segment, lowercase_code_points, path_code_points, segment_code_points, +}; use std::cmp::Ordering; +/// Python's path order on this platform; Windows paths compare their lowercased parts. pub(crate) fn compare_relative_paths(left: &str, right: &str) -> Ordering { - left.split('/').cmp(right.split('/')) + compare_paths(left, right, cfg!(windows)) +} + +/// `PurePosixPath` order, or `PureWindowsPath` order (`str.lower()` of each part) when asked. +pub(crate) fn compare_paths(left: &str, right: &str, case_insensitive: bool) -> Ordering { + if !case_insensitive && !has_raw_segment(left) && !has_raw_segment(right) { + return left.split('/').cmp(right.split('/')); + } + let key = |path: &str| -> Vec> { + path.split('/') + .map(|segment| { + let points: Vec = segment_code_points(segment); + if case_insensitive { + lowercase_code_points(&points) + } else { + points + } + }) + .collect() + }; + key(left).cmp(&key(right)) +} + +/// Python's order of `as_posix()` strings, with raw names surrogate-escaped. +pub(crate) fn compare_posix_text(left: &str, right: &str) -> Ordering { + if !has_raw_segment(left) && !has_raw_segment(right) { + return left.cmp(right); + } + path_code_points(left).cmp(&path_code_points(right)) } diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/raw_names.rs b/crates/sqlbuild-discovery/src/tree/_helpers/raw_names.rs new file mode 100644 index 0000000000..3362ef4381 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/_helpers/raw_names.rs @@ -0,0 +1,214 @@ +//! Path segments for names that are not valid UTF-8: unique, and ordered as Python orders them. + +use std::borrow::Cow; +use std::ffi::OsStr; + +/// Brackets a raw name's hex after its lossy text; no file name can contain it. +const RAW_MARKER: char = '\0'; +const PATH_SEPARATOR: u32 = '/' as u32; +const SURROGATE_ESCAPE_BASE: u32 = 0xDC00; +const ESCAPED_BYTE_FIRST: u32 = 0xDC80; +const ESCAPED_BYTE_LAST: u32 = 0xDCFF; +const BYTE_HEX_DIGITS: usize = 2; +const WIDE_HEX_DIGITS: usize = 4; +const HEX_RADIX: u32 = 16; + +/// The segment naming a listed entry whose name is not valid UTF-8: lossy text, then raw hex. +pub(crate) fn raw_segment(lossy: &str, raw: &OsStr) -> String { + format!("{lossy}{RAW_MARKER}{}{RAW_MARKER}", raw_hex(raw)) +} + +/// Whether a relative path holds a segment whose name is not valid UTF-8. +pub(crate) fn has_raw_segment(text: &str) -> bool { + text.contains(RAW_MARKER) +} + +/// Text as a user sees it: each raw name with its invalid bytes or units escaped. +pub(crate) fn display_text(text: &str) -> Cow<'_, str> { + display_text_on(text, cfg!(windows)) +} + +/// `display_text` for raw names listed on Windows (UTF-16 units) or elsewhere (bytes). +pub(crate) fn display_text_on(text: &str, windows: bool) -> Cow<'_, str> { + if !has_raw_segment(text) { + return Cow::Borrowed(text); + } + let parts: Vec<&str> = text.split(RAW_MARKER).collect(); + let mut shown = String::new(); + for (index, part) in parts.iter().enumerate() { + if index % 2 == 1 { + let points: Vec = hex_code_points(part, windows); + shown.push_str(&escaped_text(&points, windows)); + } else if index + 1 < parts.len() { + let lossy: String = lossy_text(parts[index + 1], windows); + shown.push_str(part.strip_suffix(lossy.as_str()).unwrap_or(part)); + } else { + shown.push_str(part); + } + } + Cow::Owned(shown) +} + +/// Python's code points for a raw name's hex: UTF-16 units on Windows, bytes elsewhere. +fn hex_code_points(hex: &str, windows: bool) -> Vec { + if windows { + escaped_wide(&hex_units(hex, WIDE_HEX_DIGITS)) + } else { + escaped_bytes(&hex_units(hex, BYTE_HEX_DIGITS)) + } +} + +/// The lossy name listing gave for a raw name's hex (`to_string_lossy`). +fn lossy_text(hex: &str, windows: bool) -> String { + if windows { + let units: Vec = hex_units(hex, WIDE_HEX_DIGITS) + .into_iter() + .map(|unit| u16::try_from(unit).unwrap_or(u16::MAX)) + .collect(); + String::from_utf16_lossy(&units) + } else { + let bytes: Vec = hex_units(hex, BYTE_HEX_DIGITS) + .into_iter() + .map(|byte| u8::try_from(byte).unwrap_or(u8::MAX)) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() + } +} + +/// Code points as text: an escaped byte as `\xNN`, a Windows lone surrogate as `\uXXXX`. +fn escaped_text(points: &[u32], windows: bool) -> String { + points + .iter() + .map(|point| match char::from_u32(*point) { + Some(character) => character.to_string(), + None if !windows && (ESCAPED_BYTE_FIRST..=ESCAPED_BYTE_LAST).contains(point) => { + format!("\\x{:02x}", point - SURROGATE_ESCAPE_BASE) + } + None => format!("\\u{point:04x}"), + }) + .collect() +} + +/// The code points of Python's `str` for one segment, with raw names surrogate-escaped. +pub(crate) fn segment_code_points(segment: &str) -> Vec { + segment_code_points_on(segment, cfg!(windows)) +} + +/// `segment_code_points` for raw names listed on Windows or elsewhere. +pub(crate) fn segment_code_points_on(segment: &str, windows: bool) -> Vec { + match segment.split(RAW_MARKER).nth(1) { + Some(hex) => hex_code_points(hex, windows), + None => segment.chars().map(u32::from).collect(), + } +} + +/// The code points of Python's `str` for a `/`-separated relative path. +pub(crate) fn path_code_points(relative_path: &str) -> Vec { + let mut points: Vec = Vec::new(); + for (index, segment) in relative_path.split('/').enumerate() { + if index > 0 { + points.push(PATH_SEPARATOR); + } + points.extend(segment_code_points(segment)); + } + points +} + +/// Python's `str.lower()` of code points; surrogates are not cased and stay unchanged. +pub(crate) fn lowercase_code_points(points: &[u32]) -> Vec { + let mut lowered: Vec = Vec::new(); + let mut run = String::new(); + for point in points { + match char::from_u32(*point) { + Some(character) => run.push(character), + None => { + lowered.extend(run.to_lowercase().chars().map(u32::from)); + run.clear(); + lowered.push(*point); + } + } + } + lowered.extend(run.to_lowercase().chars().map(u32::from)); + lowered +} + +/// `bytes.decode("utf-8", "surrogateescape")`: each byte of an invalid sequence becomes U+DCxx. +pub(crate) fn escaped_bytes(bytes: &[u32]) -> Vec { + let raw: Vec = bytes + .iter() + .map(|byte| u8::try_from(*byte).unwrap_or(u8::MAX)) + .collect(); + let mut points: Vec = Vec::new(); + let mut rest: &[u8] = &raw; + while !rest.is_empty() { + match std::str::from_utf8(rest) { + Ok(text) => { + points.extend(text.chars().map(u32::from)); + rest = &[]; + } + Err(error) => { + let (valid, invalid) = rest.split_at(error.valid_up_to()); + points.extend(String::from_utf8_lossy(valid).chars().map(u32::from)); + let length: usize = error.error_len().unwrap_or(invalid.len()); + points.extend( + invalid[..length] + .iter() + .map(|byte| SURROGATE_ESCAPE_BASE + u32::from(*byte)), + ); + rest = &invalid[length..]; + } + } + } + points +} + +/// A Windows name's UTF-16 units as Python's `str` keeps them, lone surrogates included. +pub(crate) fn escaped_wide(units: &[u32]) -> Vec { + let wide: Vec = units + .iter() + .map(|unit| u16::try_from(*unit).unwrap_or(u16::MAX)) + .collect(); + char::decode_utf16(wide) + .map(|decoded| match decoded { + Ok(character) => u32::from(character), + Err(error) => u32::from(error.unpaired_surrogate()), + }) + .collect() +} + +fn hex_units(hex: &str, digits: usize) -> Vec { + hex.as_bytes().chunks(digits).map(hex_value).collect() +} + +fn hex_value(digits: &[u8]) -> u32 { + let mut value: u32 = 0; + for digit in digits { + value = value * HEX_RADIX + char::from(*digit).to_digit(HEX_RADIX).unwrap_or_default(); + } + value +} + +#[cfg(unix)] +fn raw_hex(raw: &OsStr) -> String { + use std::os::unix::ffi::OsStrExt; + raw.as_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[cfg(windows)] +fn raw_hex(raw: &OsStr) -> String { + use std::os::windows::ffi::OsStrExt; + raw.encode_wide() + .map(|unit| format!("{unit:04x}")) + .collect() +} + +#[cfg(not(any(unix, windows)))] +fn raw_hex(raw: &OsStr) -> String { + raw.as_encoded_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs index 0aa7327497..4ebf756e0a 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs @@ -1,9 +1,10 @@ //! The parallel walk below one root, listing each directory once per snapshot. -use crate::models::StageDeferral; +use crate::models::StageFailure; use crate::tree::_helpers::listing::{join_relative, list_directory}; use crate::tree::models::{ProjectTree, TreeEntry}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use std::ffi::OsString; use std::sync::Arc; pub(crate) type WalkedDirectory = (String, Arc>); @@ -12,12 +13,12 @@ pub(crate) type WalkedDirectory = (String, Arc>); pub(crate) fn walk( tree: &ProjectTree, directory: &str, -) -> Result, StageDeferral> { +) -> Result, StageFailure> { let listing: Arc> = listing(tree, directory)?; let children: Vec = listing .iter() .filter(|entry| entry.is_walkable_dir) - .map(|entry| join_relative(directory, &entry.name)) + .map(|entry| join_relative(directory, &entry.segment)) .collect(); let nested: Vec> = children .into_par_iter() @@ -31,17 +32,32 @@ pub(crate) fn walk( pub(crate) fn listing( tree: &ProjectTree, directory: &str, -) -> Result>, StageDeferral> { +) -> Result>, StageFailure> { if let Some(cached) = cached_listing(tree, directory) { return Ok(cached); } - let listing: Arc> = Arc::new(list_directory(&tree.absolute(directory))?); + let listing: Arc> = Arc::new(list_directory(&tree.absolute(directory))); + remember_raw_names(tree, directory, &listing); if let Ok(mut listings) = tree.listings.lock() { listings.insert(directory.to_owned(), Arc::clone(&listing)); } Ok(listing) } +fn remember_raw_names(tree: &ProjectTree, directory: &str, listing: &[TreeEntry]) { + let mut undecodable: Vec<(String, OsString)> = Vec::new(); + for entry in listing { + if let Some(raw) = &entry.raw_name { + undecodable.push((join_relative(directory, &entry.segment), raw.clone())); + } + } + if !undecodable.is_empty() + && let Ok(mut raw_names) = tree.raw_names.write() + { + raw_names.extend(undecodable); + } +} + fn cached_listing(tree: &ProjectTree, directory: &str) -> Option>> { match tree.listings.lock() { Ok(listings) => listings.get(directory).cloned(), diff --git a/crates/sqlbuild-discovery/src/tree/main/children.rs b/crates/sqlbuild-discovery/src/tree/main/children.rs index f31bcf2130..f754e114e8 100644 --- a/crates/sqlbuild-discovery/src/tree/main/children.rs +++ b/crates/sqlbuild-discovery/src/tree/main/children.rs @@ -1,24 +1,26 @@ //! The entries of one directory, as `sorted(directory.iterdir())` sees them. -use crate::models::StageDeferral; +use crate::_helpers::reading::listing_failure; +use crate::models::StageFailure; use crate::tree::_helpers::listing::join_relative; use crate::tree::_helpers::ordering::compare_relative_paths; use crate::tree::_helpers::walking::listing; use crate::tree::models::{ProjectTree, TreeEntry}; -/// `sorted(directory.iterdir())` with relative paths; an unreadable directory defers. +/// `sorted(directory.iterdir())` with relative paths; an unreadable directory fails as in Python. pub fn children( tree: &ProjectTree, directory: &str, -) -> Result, StageDeferral> { +) -> Result, StageFailure> { if let Err(error) = std::fs::read_dir(tree.absolute(directory)) { - return Err(StageDeferral { - reason: format!("cannot list {directory}: {error}"), + return Err(StageFailure::Unlistable { + relative_path: directory.to_owned(), + error: listing_failure(&error), }); } let mut entries: Vec<(String, TreeEntry)> = listing(tree, directory)? .iter() - .map(|entry| (join_relative(directory, &entry.name), entry.clone())) + .map(|entry| (join_relative(directory, &entry.segment), entry.clone())) .collect(); entries.sort_by(|left, right| compare_relative_paths(&left.0, &right.0)); Ok(entries) diff --git a/crates/sqlbuild-discovery/src/tree/main/directories.rs b/crates/sqlbuild-discovery/src/tree/main/directories.rs index b9928dbe5c..5d9d42788a 100644 --- a/crates/sqlbuild-discovery/src/tree/main/directories.rs +++ b/crates/sqlbuild-discovery/src/tree/main/directories.rs @@ -1,10 +1,10 @@ //! `sorted(path for path in DirectorySnapshot.directories(root))` over the shared snapshot. -use crate::models::StageDeferral; +use crate::models::StageFailure; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; /// Every directory below `root`, including unwalked directory links, in Python's path order. -pub fn directories(tree: &ProjectTree, root: &str) -> Result, StageDeferral> { +pub fn directories(tree: &ProjectTree, root: &str) -> Result, StageFailure> { rglob(tree, root, |entry| entry.is_dir) } diff --git a/crates/sqlbuild-discovery/src/tree/main/display_text.rs b/crates/sqlbuild-discovery/src/tree/main/display_text.rs new file mode 100644 index 0000000000..0c21568822 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/display_text.rs @@ -0,0 +1,9 @@ +//! Path text as users see it, with names that are not valid UTF-8 shown lossily. + +use crate::tree::_helpers::raw_names; +use std::borrow::Cow; + +/// `text` with each raw path segment replaced by its lossy name. +pub fn display_text(text: &str) -> Cow<'_, str> { + raw_names::display_text(text) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs index d45f892679..643783dfed 100644 --- a/crates/sqlbuild-discovery/src/tree/main/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -1,5 +1,6 @@ pub mod children; pub mod directories; +pub mod display_text; pub mod listings; -pub mod read_tree; +pub mod path_order; pub mod rglob; diff --git a/crates/sqlbuild-discovery/src/tree/main/path_order.rs b/crates/sqlbuild-discovery/src/tree/main/path_order.rs new file mode 100644 index 0000000000..b19fb420db --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/path_order.rs @@ -0,0 +1,9 @@ +//! Python's sort order of project-relative paths. + +use crate::tree::_helpers::ordering; +use std::cmp::Ordering; + +/// Python's order of `as_posix()` strings, with names that are not UTF-8 surrogate-escaped. +pub fn compare_posix_text(left: &str, right: &str) -> Ordering { + ordering::compare_posix_text(left, right) +} diff --git a/crates/sqlbuild-discovery/src/tree/main/read_tree.rs b/crates/sqlbuild-discovery/src/tree/main/read_tree.rs deleted file mode 100644 index 00046ff610..0000000000 --- a/crates/sqlbuild-discovery/src/tree/main/read_tree.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Read every listing below a root into the shared snapshot. - -use crate::models::StageDeferral; -use crate::tree::_helpers::walking::walk; -use crate::tree::models::ProjectTree; - -/// List `root` and every real directory below it; a missing root reads nothing. -pub fn read_tree(tree: &ProjectTree, root: &str) -> Result<(), StageDeferral> { - if tree.is_dir(root) { - let _walked = walk(tree, root)?; - } - Ok(()) -} diff --git a/crates/sqlbuild-discovery/src/tree/main/rglob.rs b/crates/sqlbuild-discovery/src/tree/main/rglob.rs index ef9c44cec5..6bc1c0eb74 100644 --- a/crates/sqlbuild-discovery/src/tree/main/rglob.rs +++ b/crates/sqlbuild-discovery/src/tree/main/rglob.rs @@ -1,6 +1,6 @@ //! `sorted(root.rglob(pattern))` over the shared snapshot. -use crate::models::StageDeferral; +use crate::models::StageFailure; use crate::tree::_helpers::listing::join_relative; use crate::tree::_helpers::ordering::compare_relative_paths; use crate::tree::_helpers::walking::{WalkedDirectory, walk}; @@ -11,7 +11,7 @@ pub fn rglob( tree: &ProjectTree, root: &str, accept: impl Fn(&TreeEntry) -> bool, -) -> Result, StageDeferral> { +) -> Result, StageFailure> { if !tree.is_dir(root) { return Ok(Vec::new()); } @@ -28,6 +28,6 @@ fn accepted_paths(walked: &WalkedDirectory, accept: &impl Fn(&TreeEntry) -> bool listing .iter() .filter(|entry| accept(entry)) - .map(|entry| join_relative(directory, &entry.name)) + .map(|entry| join_relative(directory, &entry.segment)) .collect() } diff --git a/crates/sqlbuild-discovery/src/tree/models.rs b/crates/sqlbuild-discovery/src/tree/models.rs index 93ebadc3a2..90d333cbb3 100644 --- a/crates/sqlbuild-discovery/src/tree/models.rs +++ b/crates/sqlbuild-discovery/src/tree/models.rs @@ -1,13 +1,21 @@ //! One shared directory snapshot for a native discovery pass. +use crate::tree::_helpers::listing::join_relative; +use crate::tree::_helpers::raw_names::has_raw_segment; use std::collections::HashMap; +use std::ffi::OsString; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; /// One directory entry, with the same type facts as Python's `DirectorySnapshotEntry`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TreeEntry { + /// The name, with bytes that are not valid UTF-8 replaced by `\u{FFFD}`. pub name: String, + /// The real name when it is not valid UTF-8. + pub raw_name: Option, + /// The name in relative paths: `name`, or a unique segment for a raw name. + pub segment: String, /// `entry.is_dir()`, following symbolic links. pub is_dir: bool, /// `entry.is_dir(follow_symlinks=False)`: a real directory the walk descends into. @@ -19,6 +27,8 @@ pub struct TreeEntry { pub struct ProjectTree { pub(crate) directory: PathBuf, pub(crate) listings: Mutex>>>, + /// The real names of listed entries that are not valid UTF-8, by relative path. + pub(crate) raw_names: RwLock>, } impl ProjectTree { @@ -27,15 +37,48 @@ impl ProjectTree { Self { directory: directory.to_path_buf(), listings: Mutex::new(HashMap::new()), + raw_names: RwLock::new(HashMap::new()), } } - /// The absolute path of a `/`-separated project-relative path. + /// The absolute path of a `/`-separated project-relative path, using real listed names. pub fn absolute(&self, relative_path: &str) -> PathBuf { - relative_path - .split('/') - .filter(|part| !part.is_empty()) - .fold(self.directory.clone(), |path, part| path.join(part)) + match self.raw_relative(relative_path) { + Some(raw) => self.directory.join(raw), + None => relative_path + .split('/') + .filter(|part| !part.is_empty()) + .fold(self.directory.clone(), |path, part| path.join(part)), + } + } + + /// The real relative path when a listed component of `relative_path` is not valid UTF-8. + pub fn raw_relative(&self, relative_path: &str) -> Option { + if !has_raw_segment(relative_path) { + return None; + } + let Ok(raw_names) = self.raw_names.read() else { + return None; + }; + let mut prefix = String::new(); + let mut raw = PathBuf::new(); + let mut undecodable = false; + for part in relative_path.split('/').filter(|part| !part.is_empty()) { + prefix = join_relative(&prefix, part); + match raw_names.get(&prefix) { + Some(name) => { + undecodable = true; + raw.push(name); + } + None => raw.push(part), + } + } + undecodable.then_some(raw) + } + + /// Whether a listed component of `relative_path` is not valid UTF-8. + pub fn is_undecodable(&self, relative_path: &str) -> bool { + has_raw_segment(relative_path) } /// Python's `Path.is_dir()`: follows symbolic links and reports any failure as `false`. diff --git a/crates/sqlbuild-discovery/src/tree/tests.rs b/crates/sqlbuild-discovery/src/tree/tests.rs index 6993fa3916..5775f2500c 100644 --- a/crates/sqlbuild-discovery/src/tree/tests.rs +++ b/crates/sqlbuild-discovery/src/tree/tests.rs @@ -1,6 +1,21 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_ordering.rs"] +mod ordering; +#[path = "tests/test_os_failures.rs"] +mod os_failures; #[path = "tests/test_project_tree.rs"] mod project_tree; +#[path = "tests/test_raw_segments.rs"] +mod raw_segments; #[path = "tests/test_types.rs"] mod test_types; +#[cfg(unix)] +#[path = "tests/test_undecodable_names.rs"] +mod undecodable_names; +#[cfg(windows)] +#[path = "tests/test_windows_junctions.rs"] +mod windows_junctions; +#[cfg(windows)] +#[path = "tests/test_windows_undecodable_names.rs"] +mod windows_undecodable_names; diff --git a/crates/sqlbuild-discovery/src/tree/tests/helpers.rs b/crates/sqlbuild-discovery/src/tree/tests/helpers.rs index fa88ccc6d6..9a64c60a36 100644 --- a/crates/sqlbuild-discovery/src/tree/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/tree/tests/helpers.rs @@ -36,3 +36,107 @@ pub(super) fn globbed_paths(test_case: &GlobTestCase) -> Vec { }) .expect("walk") } + +/// Walk the case's raw-named files: displayed paths, read failures and each resolved file's text. +#[cfg(unix)] +pub(super) fn undecodable_walk( + test_case: &crate::tree::tests::test_types::UndecodableWalkTestCase, +) -> (Vec, Vec>, Vec) { + use std::os::unix::ffi::OsStrExt; + let project = tempfile::tempdir().expect("temporary project"); + for (index, relative_path) in test_case.files.iter().enumerate() { + let path = project + .path() + .join(std::ffi::OsStr::from_bytes(relative_path)); + fs::create_dir_all(path.parent().expect("parent directory")).expect("directories"); + fs::write(path, format!("file {index}")).expect("file"); + } + let tree = ProjectTree::new(project.path()); + let root = crate::models::ProjectRoot { + directory: project.path().to_path_buf(), + display_prefix: "project/".to_owned(), + }; + let paths: Vec = + rglob(&tree, "models", |entry| entry.name.ends_with(".sql")).expect("walk"); + let failures: Vec> = paths + .iter() + .map(|path| { + crate::_helpers::reading::undecodable_path_failure(&root, &tree, path) + .map(|failure| failure.message) + }) + .collect(); + let contents: Vec = paths + .iter() + .map(|path| fs::read_to_string(tree.absolute(path)).expect("resolved file")) + .collect(); + let shown: Vec = paths + .iter() + .map(|path| crate::tree::main::display_text::display_text(path).into_owned()) + .collect(); + (shown, failures, contents) +} + +/// Walk `models/` through a junction `models/linked` to a directory holding `orders.sql`. +#[cfg(windows)] +pub(super) fn junction_walk() -> Vec { + let project = tempfile::tempdir().expect("temporary project"); + let target = project.path().join("elsewhere"); + fs::create_dir_all(&target).expect("target directory"); + fs::write(target.join("orders.sql"), "MODEL ();\nSELECT 1").expect("file"); + fs::create_dir_all(project.path().join("models")).expect("models directory"); + junction::create(&target, project.path().join("models").join("linked")).expect("junction"); + rglob(&ProjectTree::new(project.path()), "models", |entry| { + entry.name.ends_with(".sql") + }) + .expect("walk") +} + +/// Python's code points for a segment whose raw name was listed on Windows or elsewhere. +pub(super) fn segment_points(segment: &str, windows: bool) -> Vec { + crate::tree::_helpers::raw_names::segment_code_points_on(segment, windows) +} + +/// Owned copies of borrowed texts. +pub(super) fn owned<'a>(texts: impl Iterator) -> Vec { + texts.map(str::to_owned).collect() +} + +/// Walk files named by UTF-16 units: displayed paths, whether each fails when read, and its text. +#[cfg(windows)] +pub(super) fn wide_name_walk( + test_case: &crate::tree::tests::test_types::WideNameWalkTestCase, +) -> (Vec, Vec, Vec) { + use std::os::windows::ffi::OsStringExt; + let project = tempfile::tempdir().expect("temporary project"); + let models = project.path().join("models"); + fs::create_dir_all(&models).expect("models directory"); + for (index, name) in test_case.names.iter().enumerate() { + fs::write( + models.join(std::ffi::OsString::from_wide(name)), + format!("file {index}"), + ) + .expect("file"); + } + let tree = ProjectTree::new(project.path()); + let root = crate::models::ProjectRoot { + directory: project.path().to_path_buf(), + display_prefix: String::new(), + }; + let paths: Vec = + rglob(&tree, "models", |entry| entry.name.ends_with(".sql")).expect("walk"); + let failures: Vec = paths + .iter() + .map(|path| { + crate::_helpers::reading::undecodable_path_failure(&root, &tree, path).is_some() + }) + .collect(); + let contents: Vec = paths + .iter() + .map(|path| fs::read_to_string(tree.absolute(path)).expect("resolved file")) + .collect(); + let shown: Vec = paths + .iter() + .map(|path| crate::tree::main::display_text::display_text(path).into_owned()) + .collect(); + (shown, failures, contents) +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_ordering.rs b/crates/sqlbuild-discovery/src/tree/tests/test_ordering.rs new file mode 100644 index 0000000000..f24d7382bf --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_ordering.rs @@ -0,0 +1,51 @@ +use crate::tree::_helpers::ordering::compare_paths; +use crate::tree::tests::test_types::PathOrderTestCase; + +#[test] +fn given_paths_when_sorting_then_order_matches_python_path_order_per_platform() { + let paths: &[&str] = &[ + "models/b.sql", + "models/A.sql", + "models/a-c.sql", + "models/a/b.sql", + ]; + let test_cases = [ + PathOrderTestCase { + description: "PurePosixPath compares parts by code point", + case_insensitive: false, + paths, + expected_paths: &[ + "models/A.sql", + "models/a/b.sql", + "models/a-c.sql", + "models/b.sql", + ], + }, + PathOrderTestCase { + description: "PureWindowsPath compares lowercased parts", + case_insensitive: true, + paths, + expected_paths: &[ + "models/a/b.sql", + "models/a-c.sql", + "models/A.sql", + "models/b.sql", + ], + }, + PathOrderTestCase { + description: "lowercasing is Unicode-aware", + case_insensitive: true, + paths: &["models/É.sql", "models/f.sql", "models/é/x.sql"], + expected_paths: &["models/f.sql", "models/é/x.sql", "models/É.sql"], + }, + ]; + for test_case in test_cases { + let mut sorted: Vec<&str> = test_case.paths.to_vec(); + sorted.sort_by(|left, right| compare_paths(left, right, test_case.case_insensitive)); + assert_eq!( + sorted, test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_os_failures.rs b/crates/sqlbuild-discovery/src/tree/tests/test_os_failures.rs new file mode 100644 index 0000000000..4820b14773 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_os_failures.rs @@ -0,0 +1,146 @@ +use crate::_helpers::reading::{os_listing_failure, os_read_failure, windows_errno}; +use crate::models::ReadFailure; +use crate::tree::tests::test_types::{OsFailureTestCase, WindowsErrnoTestCase}; + +#[test] +fn given_win32_errors_when_mapping_then_errno_matches_cpython_winerror_to_errno() { + let test_cases = [ + WindowsErrnoTestCase { + description: "access denied is EACCES", + winerror: 5, + expected_errno: 13, + }, + WindowsErrnoTestCase { + description: "a sharing violation is EACCES", + winerror: 32, + expected_errno: 13, + }, + WindowsErrnoTestCase { + description: "a missing path is ENOENT", + winerror: 3, + expected_errno: 2, + }, + WindowsErrnoTestCase { + description: "a missing file is ENOENT", + winerror: 2, + expected_errno: 2, + }, + WindowsErrnoTestCase { + description: "a file name that is not a directory is ENOTDIR", + winerror: 267, + expected_errno: 20, + }, + WindowsErrnoTestCase { + description: "a FACILITY_WIN32 HRESULT unwraps to its error", + winerror: 0x8007_0005_u32.cast_signed(), + expected_errno: 13, + }, + WindowsErrnoTestCase { + description: "a Winsock access error is its errno", + winerror: 10013, + expected_errno: 13, + }, + WindowsErrnoTestCase { + description: "an unmapped error is EINVAL", + winerror: 1234, + expected_errno: 22, + }, + ]; + for test_case in test_cases { + assert_eq!( + windows_errno(test_case.winerror), + test_case.expected_errno, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_os_errors_when_reading_then_each_platform_matches_python() { + let test_cases = [ + OsFailureTestCase { + description: "a POSIX read keeps its errno", + code: Some(13), + message: "Permission denied (os error 13)", + windows: false, + expected_errno: Some(13), + expected_winerror: None, + expected_message: "Permission denied (os error 13)", + }, + OsFailureTestCase { + description: "a Windows read reports the C runtime errno", + code: Some(32), + message: "The process cannot access the file. (os error 32)", + windows: true, + expected_errno: Some(13), + expected_winerror: None, + expected_message: "The process cannot access the file. (os error 32)", + }, + OsFailureTestCase { + description: "a read without a code keeps only its message", + code: None, + message: "stream did not contain valid UTF-8", + windows: true, + expected_errno: None, + expected_winerror: None, + expected_message: "stream did not contain valid UTF-8", + }, + ]; + for test_case in test_cases { + assert_eq!( + os_read_failure( + test_case.code, + test_case.message.to_owned(), + test_case.windows + ), + ReadFailure::Io { + errno: test_case.expected_errno, + winerror: test_case.expected_winerror, + message: test_case.expected_message.to_owned(), + }, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_listing_errors_when_reporting_then_windows_keeps_the_win32_error() { + let test_cases = [ + OsFailureTestCase { + description: "a POSIX listing keeps its errno", + code: Some(13), + message: "Permission denied (os error 13)", + windows: false, + expected_errno: Some(13), + expected_winerror: None, + expected_message: "Permission denied (os error 13)", + }, + OsFailureTestCase { + description: "a Windows listing keeps the Win32 error and the bare system message", + code: Some(5), + message: "Access is denied. (os error 5)", + windows: true, + expected_errno: None, + expected_winerror: Some(5), + expected_message: "Access is denied", + }, + ]; + for test_case in test_cases { + assert_eq!( + os_listing_failure( + test_case.code, + test_case.message.to_owned(), + test_case.windows + ), + ReadFailure::Io { + errno: test_case.expected_errno, + winerror: test_case.expected_winerror, + message: test_case.expected_message.to_owned(), + }, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs b/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs index 11b214bcb6..a6d08e20a4 100644 --- a/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs +++ b/crates/sqlbuild-discovery/src/tree/tests/test_project_tree.rs @@ -31,14 +31,6 @@ fn given_project_layouts_when_globbing_then_paths_match_python_rglob_and_sort() suffix: ".sql", expected_paths: &[], }, - GlobTestCase { - description: "a directory link matches by name but is not walked", - files: &["elsewhere/inner.sql", "models/x.txt"], - directories: &[], - links: &[("models/linked.sql", "elsewhere")], - suffix: ".sql", - expected_paths: &["models/linked.sql"], - }, ]; for test_case in test_cases { assert_eq!( @@ -49,3 +41,24 @@ fn given_project_layouts_when_globbing_then_paths_match_python_rglob_and_sort() ); } } + +#[cfg(unix)] +#[test] +fn given_a_directory_link_when_globbing_then_it_matches_by_name_but_is_not_walked() { + let test_cases = [GlobTestCase { + description: "a directory link matches by name but is not walked", + files: &["elsewhere/inner.sql", "models/x.txt"], + directories: &[], + links: &[("models/linked.sql", "elsewhere")], + suffix: ".sql", + expected_paths: &["models/linked.sql"], + }]; + for test_case in test_cases { + assert_eq!( + globbed_paths(&test_case), + test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_raw_segments.rs b/crates/sqlbuild-discovery/src/tree/tests/test_raw_segments.rs new file mode 100644 index 0000000000..566f2b8eb7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_raw_segments.rs @@ -0,0 +1,82 @@ +use crate::tree::_helpers::raw_names::{display_text_on, escaped_wide}; +use crate::tree::tests::helpers::segment_points; +use crate::tree::tests::test_types::{RawSegmentTestCase, WideUnitsTestCase}; + +#[test] +fn given_raw_segments_when_displaying_and_decoding_then_python_sees_its_own_string() { + let test_cases = [ + RawSegmentTestCase { + description: "a valid name is unchanged", + windows: false, + segment: "orders.sql", + expected_display: "orders.sql", + expected_code_points: &[0x6F, 0x72, 0x64, 0x65, 0x72, 0x73, 0x2E, 0x73, 0x71, 0x6C], + }, + RawSegmentTestCase { + description: "a POSIX invalid byte is surrogate-escaped and shown as a byte", + windows: false, + segment: "a\u{FFFD}\u{0}61e9\u{0}", + expected_display: "a\\xe9", + expected_code_points: &[0x61, 0xDCE9], + }, + RawSegmentTestCase { + description: "a POSIX cut sequence escapes each of its bytes", + windows: false, + segment: "\u{FFFD}\u{0}e282\u{0}", + expected_display: "\\xe2\\x82", + expected_code_points: &[0xDCE2, 0xDC82], + }, + RawSegmentTestCase { + description: "a Windows lone high surrogate is kept and shown as a unit", + windows: true, + segment: "a\u{FFFD}.sql\u{0}0061d800002e00730071006c\u{0}", + expected_display: "a\\ud800.sql", + expected_code_points: &[0x61, 0xD800, 0x2E, 0x73, 0x71, 0x6C], + }, + RawSegmentTestCase { + description: "a Windows lone low surrogate is a unit, never a byte", + windows: true, + segment: "a\u{FFFD}\u{0}0061dce9\u{0}", + expected_display: "a\\udce9", + expected_code_points: &[0x61, 0xDCE9], + }, + RawSegmentTestCase { + description: "a Windows pair outside the BMP decodes to one code point", + windows: true, + segment: "\u{1F600}\u{0}d83dde00\u{0}", + expected_display: "\u{1F600}", + expected_code_points: &[0x1F600], + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + display_text_on(test_case.segment, test_case.windows).into_owned(), + segment_points(test_case.segment, test_case.windows) + ), + ( + test_case.expected_display.to_owned(), + test_case.expected_code_points.to_vec() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_windows_units_when_decoding_then_lone_surrogates_are_kept() { + let test_cases = [WideUnitsTestCase { + description: "a lone surrogate stays as Python's str keeps it", + units: &[0x61, 0xD800, 0xD83D, 0xDE00], + expected_code_points: &[0x61, 0xD800, 0x1F600], + }]; + for test_case in test_cases { + assert_eq!( + escaped_wide(test_case.units), + test_case.expected_code_points, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_types.rs b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs index f5f6278f16..bae31672cf 100644 --- a/crates/sqlbuild-discovery/src/tree/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs @@ -7,3 +7,69 @@ pub(super) struct GlobTestCase { pub(super) suffix: &'static str, pub(super) expected_paths: &'static [&'static str], } + +pub(super) struct PathOrderTestCase { + pub(super) description: &'static str, + pub(super) case_insensitive: bool, + pub(super) paths: &'static [&'static str], + pub(super) expected_paths: &'static [&'static str], +} + +pub(super) struct WindowsErrnoTestCase { + pub(super) description: &'static str, + pub(super) winerror: i32, + pub(super) expected_errno: i32, +} + +pub(super) struct OsFailureTestCase { + pub(super) description: &'static str, + pub(super) code: Option, + pub(super) message: &'static str, + pub(super) windows: bool, + pub(super) expected_errno: Option, + pub(super) expected_winerror: Option, + pub(super) expected_message: &'static str, +} + +/// Files written by raw byte path below a project whose `models/` is walked for `*.sql`. +#[cfg(unix)] +pub(super) struct UndecodableWalkTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static [u8]], + pub(super) expected_paths: &'static [&'static str], + pub(super) expected_failures: &'static [Option<&'static str>], + /// `file N` for the Nth written file, in walk order. + pub(super) expected_contents: &'static [&'static str], +} + +/// A junction `models/linked` to a directory holding `orders.sql`, walked for `*.sql`. +#[cfg(windows)] +pub(super) struct JunctionWalkTestCase { + pub(super) description: &'static str, + pub(super) expected_paths: &'static [&'static str], +} + +pub(super) struct RawSegmentTestCase { + pub(super) description: &'static str, + /// Whether the raw name was listed on Windows (UTF-16 unit hex) rather than POSIX (byte hex). + pub(super) windows: bool, + pub(super) segment: &'static str, + pub(super) expected_display: &'static str, + pub(super) expected_code_points: &'static [u32], +} + +pub(super) struct WideUnitsTestCase { + pub(super) description: &'static str, + pub(super) units: &'static [u32], + pub(super) expected_code_points: &'static [u32], +} + +/// Files named by UTF-16 units below `models/`, walked for `*.sql` on Windows. +#[cfg(windows)] +pub(super) struct WideNameWalkTestCase { + pub(super) description: &'static str, + pub(super) names: &'static [&'static [u16]], + pub(super) expected_paths: &'static [&'static str], + pub(super) expected_failures: &'static [bool], + pub(super) expected_contents: &'static [&'static str], +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_undecodable_names.rs b/crates/sqlbuild-discovery/src/tree/tests/test_undecodable_names.rs new file mode 100644 index 0000000000..3e5cb723c1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_undecodable_names.rs @@ -0,0 +1,71 @@ +use crate::tree::tests::helpers::{owned, undecodable_walk}; +use crate::tree::tests::test_types::UndecodableWalkTestCase; + +const RENAME: &str = " is not valid UTF-8; rename it so SQLBuild can read it"; + +#[test] +fn given_names_that_are_not_utf8_when_walking_then_each_file_keeps_its_own_identity() { + let test_cases = [ + UndecodableWalkTestCase { + description: "lossy names sort, resolve to real files and fail only when read", + files: &[ + b"models/orders.sql", + b"models/notes\xe9.txt", + b"models/old\xe9/inner.sql", + b"models/caf\xe9.sql", + ], + expected_paths: &[ + "models/caf\\xe9.sql", + "models/old\\xe9/inner.sql", + "models/orders.sql", + ], + expected_failures: &[ + Some("Project path project/models/caf\\xe9.sql"), + Some("Project path project/models/old\\xe9/inner.sql"), + None, + ], + expected_contents: &["file 3", "file 2", "file 0"], + }, + UndecodableWalkTestCase { + description: "names sharing a lossy name stay distinct and a valid name is not blamed", + files: &[ + b"models/st\xe9.sql", + b"models/st\xe8.sql", + b"models/st\xef\xbf\xbd.sql", + b"models/a\xe9/x.sql", + b"models/a\xe8/y.sql", + ], + expected_paths: &[ + "models/a\\xe8/y.sql", + "models/a\\xe9/x.sql", + "models/st\\xe8.sql", + "models/st\\xe9.sql", + "models/st\u{FFFD}.sql", + ], + expected_failures: &[ + Some("Project path project/models/a\\xe8/y.sql"), + Some("Project path project/models/a\\xe9/x.sql"), + Some("Project path project/models/st\\xe8.sql"), + Some("Project path project/models/st\\xe9.sql"), + None, + ], + expected_contents: &["file 4", "file 3", "file 1", "file 0", "file 2"], + }, + ]; + for test_case in test_cases { + assert_eq!( + undecodable_walk(&test_case), + ( + owned(test_case.expected_paths.iter().copied()), + test_case + .expected_failures + .iter() + .map(|failure| failure.map(|prefix| format!("{prefix}{RENAME}"))) + .collect::>>(), + owned(test_case.expected_contents.iter().copied()), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_windows_junctions.rs b/crates/sqlbuild-discovery/src/tree/tests/test_windows_junctions.rs new file mode 100644 index 0000000000..17d24a37a8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_windows_junctions.rs @@ -0,0 +1,18 @@ +use crate::tree::tests::helpers::junction_walk; +use crate::tree::tests::test_types::JunctionWalkTestCase; + +#[test] +fn given_a_junction_when_walking_then_it_is_walked_like_python_walks_a_directory() { + let test_cases = [JunctionWalkTestCase { + description: "a junction is a real directory to Python's os.scandir", + expected_paths: &["models/linked/orders.sql"], + }]; + for test_case in test_cases { + assert_eq!( + junction_walk(), + test_case.expected_paths, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_windows_undecodable_names.rs b/crates/sqlbuild-discovery/src/tree/tests/test_windows_undecodable_names.rs new file mode 100644 index 0000000000..f4ab9a56e5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_windows_undecodable_names.rs @@ -0,0 +1,35 @@ +use crate::tree::tests::helpers::{owned, wide_name_walk}; +use crate::tree::tests::test_types::WideNameWalkTestCase; + +#[test] +fn given_windows_names_with_lone_surrogates_when_walking_then_each_file_keeps_its_identity() { + let test_cases = [WideNameWalkTestCase { + description: "lone surrogates sharing a lossy name resolve to their own files", + names: &[ + &[0x6F, 0x72, 0x64, 0x65, 0x72, 0x73, 0x2E, 0x73, 0x71, 0x6C], + &[0x61, 0xDCE9, 0x2E, 0x73, 0x71, 0x6C], + &[0x61, 0xD800, 0x2E, 0x73, 0x71, 0x6C], + &[0x61, 0xFFFD, 0x2E, 0x73, 0x71, 0x6C], + ], + expected_paths: &[ + "models/a\\ud800.sql", + "models/a\\udce9.sql", + "models/a\u{FFFD}.sql", + "models/orders.sql", + ], + expected_failures: &[true, true, false, false], + expected_contents: &["file 2", "file 1", "file 3", "file 0"], + }]; + for test_case in test_cases { + assert_eq!( + wide_name_walk(&test_case), + ( + owned(test_case.expected_paths.iter().copied()), + test_case.expected_failures.to_vec(), + owned(test_case.expected_contents.iter().copied()), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/failures.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/failures.rs new file mode 100644 index 0000000000..bad454a2be --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/failures.rs @@ -0,0 +1,38 @@ +//! The discovery failure a YAML document the native reader rejects is reported as. + +use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_config::errors::{ConfigError, ConfigErrorKind}; + +const UNSUPPORTED_HELP: &str = "SQLBuild reads plain YAML: block and flow mappings and \ + sequences, plain and quoted scalars, literal and folded block \ + scalars, anchors, aliases and merge keys. Rewrite this part in \ + one of those forms."; + +/// The failure of a document `file_path` names, at the error's position where known. +pub(crate) fn yaml_failure( + file_path: &str, + kind: FailureKind, + error: ConfigError, +) -> DiscoveryFailure { + let position: String = match (error.line, error.column) { + (Some(line), Some(column)) => format!(" at line {line}, column {column}"), + _ => String::new(), + }; + match error.kind { + ConfigErrorKind::Unsupported => DiscoveryFailure { + kind, + message: format!( + "{file_path} uses YAML that SQLBuild does not support{position}: {}", + error.message + ), + help: Some(UNSUPPORTED_HELP.to_owned()), + }, + _ => DiscoveryFailure::new( + kind, + format!( + "{file_path} contains invalid YAML{position}: {}", + error.message + ), + ), + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs new file mode 100644 index 0000000000..6540a7ca43 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod failures; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs index 85bd8a1291..db440cdffb 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_files.rs @@ -1,19 +1,22 @@ -//! Read and load the given YAML files in parallel, deferring each file Python must load. +//! Read and load the given YAML files in parallel. use crate::_helpers::pool::discovery_pool; use crate::_helpers::reading::read_authored_text; -use crate::models::StageDeferral; +use crate::models::{FailureKind, ProjectRoot, StageFailure}; use crate::tree::models::ProjectTree; +use crate::yaml_files::_helpers::failures::yaml_failure; use crate::yaml_files::models::YamlFileOutcome; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_config::yaml::main::safe_load::safe_load; -/// Each file's outcome, in the order of `relative_paths`. +/// Each file's outcome, in the order of `relative_paths`; failures are raised as `kind`. pub fn load_yaml_files( + root: &ProjectRoot, tree: &ProjectTree, relative_paths: &[String], -) -> Result, StageDeferral> { - let pool = discovery_pool().map_err(|reason| StageDeferral { reason })?; + kind: FailureKind, +) -> Result, StageFailure> { + let pool = discovery_pool().map_err(StageFailure::Internal)?; Ok(pool.install(|| { relative_paths .par_iter() @@ -21,9 +24,13 @@ pub fn load_yaml_files( |relative_path| match read_authored_text(&tree.absolute(relative_path)) { Ok(contents) => match safe_load(&contents) { Ok(value) => YamlFileOutcome::Loaded { contents, value }, - Err(_deferred) => YamlFileOutcome::LoadInPython { contents }, + Err(error) => YamlFileOutcome::Failed(yaml_failure( + &root.display_path(relative_path), + kind, + error, + )), }, - Err(_unreadable) => YamlFileOutcome::Unreadable, + Err(failure) => YamlFileOutcome::Unreadable(failure), }, ) .collect() diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_text.rs b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_text.rs new file mode 100644 index 0000000000..635525f483 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/load_yaml_text.rs @@ -0,0 +1,15 @@ +//! Load one in-memory YAML document as a discovered file would load. + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::yaml_files::_helpers::failures::yaml_failure; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::yaml::main::safe_load::safe_load; + +/// The document's value, or the failure raised as `kind` naming `file_path`. +pub fn load_yaml_text( + file_path: &str, + text: &str, + kind: FailureKind, +) -> Result { + safe_load(text).map_err(|error| yaml_failure(file_path, kind, error)) +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs index d96b3a0c0b..5155a68f04 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -1 +1,2 @@ pub mod load_yaml_files; +pub mod load_yaml_text; diff --git a/crates/sqlbuild-discovery/src/yaml_files/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/mod.rs index 1a70a48981..e6e5a52eb4 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod _helpers; pub mod main; pub mod models; diff --git a/crates/sqlbuild-discovery/src/yaml_files/models.rs b/crates/sqlbuild-discovery/src/yaml_files/models.rs index 7aa974d0d5..79025f276b 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/models.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/models.rs @@ -1,16 +1,17 @@ //! Native outcomes of reading and loading authored YAML files. +use crate::models::{DiscoveryFailure, ReadFailure}; use sqlbuild_config::models::ConfigValue; -/// One YAML file read and loaded as PyYAML `safe_load` would, or what Python must redo. +/// One YAML file read and loaded as YAML 1.1 `safe_load`, or why it could not be. #[derive(Clone, Debug, PartialEq)] pub enum YamlFileOutcome { Loaded { contents: String, value: ConfigValue, }, - /// Read, but the native loader cannot reproduce PyYAML here; Python loads the contents. - LoadInPython { contents: String }, - /// The bytes could not be read or decoded; Python re-reads the file to raise its error. - Unreadable, + /// Invalid YAML, or YAML outside the forms SQLBuild reads. + Failed(DiscoveryFailure), + /// The file could not be read or decoded as Python's `read_text` would. + Unreadable(ReadFailure), } diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs index 28a1ca493a..f7330423e5 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/helpers.rs @@ -1,8 +1,39 @@ +use crate::models::{DiscoveryFailure, FailureKind, ProjectRoot}; +use crate::tree::models::ProjectTree; +use crate::yaml_files::main::load_yaml_files::load_yaml_files; +use crate::yaml_files::main::load_yaml_text::load_yaml_text; use crate::yaml_files::models::YamlFileOutcome; -/// An expected "Python loads the contents" outcome; only its variant is compared. -pub(super) fn python() -> YamlFileOutcome { - YamlFileOutcome::LoadInPython { - contents: String::new(), - } +pub(super) const FILE_PATH: &str = "project/sources/orders.yml"; + +/// The outcome of loading one file holding `contents`, with an empty display prefix. +pub(super) fn file_outcome(contents: &[u8]) -> YamlFileOutcome { + let project = tempfile::tempdir().expect("temporary project"); + std::fs::write(project.path().join("file.yml"), contents).expect("file"); + let root = ProjectRoot { + directory: project.path().to_path_buf(), + display_prefix: String::new(), + }; + let mut outcomes: Vec = load_yaml_files( + &root, + &ProjectTree::new(project.path()), + &["file.yml".to_owned()], + FailureKind::Source, + ) + .expect("no stage failure"); + outcomes.pop().expect("one outcome") +} + +/// The failure message and help of loading `text`, or `None` when it loads. +pub(super) fn text_failure(text: &str) -> Option<(String, Option)> { + load_yaml_text(FILE_PATH, text, FailureKind::Source) + .err() + .map(|failure: DiscoveryFailure| (failure.message, failure.help)) +} + +/// The expected failure in the owned shape `text_failure` returns. +pub(super) fn owned_failure( + expected: Option<(&'static str, Option<&'static str>)>, +) -> Option<(String, Option)> { + expected.map(|(message, help)| (message.to_owned(), help.map(str::to_owned))) } diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs index 9f019458a0..d488a3c044 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_load_yaml_files.rs @@ -1,17 +1,15 @@ -use crate::tree::models::ProjectTree; -use crate::yaml_files::main::load_yaml_files::load_yaml_files; +use crate::models::ReadFailure; use crate::yaml_files::models::YamlFileOutcome; -use crate::yaml_files::tests::test_types::YamlFileTestCase; - -use crate::yaml_files::tests::helpers::python; +use crate::yaml_files::tests::helpers::{file_outcome, owned_failure, text_failure}; +use crate::yaml_files::tests::test_types::{YamlFileTestCase, YamlTextTestCase}; use sqlbuild_config::models::ConfigValue; use std::mem::discriminant; #[test] -fn given_yaml_files_when_loading_then_each_file_loads_or_defers() { +fn given_yaml_files_when_loading_then_each_file_loads_or_fails() { let test_cases = [ YamlFileTestCase { - description: "a plain mapping loads natively", + description: "a plain mapping loads", contents: b"sources:\n - name: orders\n", expected_outcome: YamlFileOutcome::Loaded { contents: String::new(), @@ -19,29 +17,129 @@ fn given_yaml_files_when_loading_then_each_file_loads_or_defers() { }, }, YamlFileTestCase { - description: "invalid YAML is left to Python for its exact error", + description: "invalid YAML fails", contents: b"sources: [\n", - expected_outcome: python(), + expected_outcome: YamlFileOutcome::Failed(crate::models::DiscoveryFailure::new( + crate::models::FailureKind::Source, + String::new(), + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + discriminant(&file_outcome(test_case.contents)), + discriminant(&test_case.expected_outcome), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_undecodable_yaml_file_when_loading_then_python_decode_error_is_reported() { + let test_cases = [ + YamlFileTestCase { + description: "an invalid start byte", + contents: b"name: \xff\n", + expected_outcome: YamlFileOutcome::Unreadable(ReadFailure::Decode { + bytes: b"name: \xff\n".to_vec(), + start: 6, + end: 7, + reason: "invalid start byte", + }), }, YamlFileTestCase { - description: "a Python-only tag is left to Python", - contents: b"value: !!set {a: null}\n", - expected_outcome: python(), + description: "a truncated sequence at the end", + contents: b"name: \xe2\x82", + expected_outcome: YamlFileOutcome::Unreadable(ReadFailure::Decode { + bytes: b"name: \xe2\x82".to_vec(), + start: 6, + end: 8, + reason: "unexpected end of data", + }), }, YamlFileTestCase { - description: "undecodable bytes are re-read by Python", - contents: b"name: \xff\n", - expected_outcome: YamlFileOutcome::Unreadable, + description: "an invalid continuation byte", + contents: b"name: \xe2\x82x", + expected_outcome: YamlFileOutcome::Unreadable(ReadFailure::Decode { + bytes: b"name: \xe2\x82x".to_vec(), + start: 6, + end: 8, + reason: "invalid continuation byte", + }), + }, + ]; + for test_case in test_cases { + assert_eq!( + file_outcome(test_case.contents), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_yaml_text_when_loading_then_failures_name_file_position_and_construct() { + let test_cases = [ + YamlTextTestCase { + description: "common forms load", + text: "sources:\n - name: orders\n description: |\n Raw orders.\n meta: &m {a: 1}\n other:\n <<: *m\n", + expected_failure: None, + }, + YamlTextTestCase { + description: "an unclosed flow sequence is invalid YAML", + text: "sources: [\n", + expected_failure: Some(( + "project/sources/orders.yml contains invalid YAML at line 2, column 1: while \ + parsing a node, did not find expected node content", + None, + )), + }, + YamlTextTestCase { + description: "a set tag is unsupported at its node", + text: "sources:\n - name: orders\n meta: !!set {a: null}\n", + expected_failure: Some(( + "project/sources/orders.yml uses YAML that SQLBuild does not support at line 3, \ + column 17: values tagged !!set", + Some( + "SQLBuild reads plain YAML: block and flow mappings and sequences, plain and \ + quoted scalars, literal and folded block scalars, anchors, aliases and merge \ + keys. Rewrite this part in one of those forms.", + ), + )), + }, + YamlTextTestCase { + description: "a tab is unsupported at its character", + text: "sources:\n - name: \"a\tb\"\n", + expected_failure: Some(( + "project/sources/orders.yml uses YAML that SQLBuild does not support at line 2, \ + column 13: tab characters", + Some( + "SQLBuild reads plain YAML: block and flow mappings and sequences, plain and \ + quoted scalars, literal and folded block scalars, anchors, aliases and merge \ + keys. Rewrite this part in one of those forms.", + ), + )), + }, + YamlTextTestCase { + description: "a directive is unsupported at its line", + text: "%YAML 1.1\n---\nsources: []\n", + expected_failure: Some(( + "project/sources/orders.yml uses YAML that SQLBuild does not support at line 1, \ + column 1: %YAML and %TAG directives", + Some( + "SQLBuild reads plain YAML: block and flow mappings and sequences, plain and \ + quoted scalars, literal and folded block scalars, anchors, aliases and merge \ + keys. Rewrite this part in one of those forms.", + ), + )), }, ]; for test_case in test_cases { - let project = tempfile::tempdir().expect("temporary project"); - std::fs::write(project.path().join("file.yml"), test_case.contents).expect("file"); - let outcomes = load_yaml_files(&ProjectTree::new(project.path()), &["file.yml".to_owned()]) - .expect("no deferral"); assert_eq!( - outcomes.iter().map(discriminant).collect::>(), - vec![discriminant(&test_case.expected_outcome)], + text_failure(test_case.text), + owned_failure(test_case.expected_failure), "{}", test_case.description ); diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs index 99eba3e5a5..f9cb105aab 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -3,6 +3,12 @@ use crate::yaml_files::models::YamlFileOutcome; pub(super) struct YamlFileTestCase { pub(super) description: &'static str, pub(super) contents: &'static [u8], - /// Only the variant is compared. pub(super) expected_outcome: YamlFileOutcome, } + +pub(super) struct YamlTextTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + /// The failure message and help, or `None` when the document loads. + pub(super) expected_failure: Option<(&'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 8826d44899..5de24615c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -1,27 +1,34 @@ //! Native discovery results as the plain data the Python discovery facade materialises. +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyTuple; +use pyo3::types::{PyBytes, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_core::text::models::PythonText; use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; use sqlbuild_discovery::declarations::models::{ - DeclarationFileFact, DeclarationGroup, DeclarationLayout, + DeclarationFileFact, DeclarationGroup, DeclarationKind, DeclarationLayout, }; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{ - DiscoveredFile, DiscoveryFailure, FileOutcome, LineColumnSpan, ProjectRoot, StageDeferral, + DiscoveredFile, DiscoveryFailure, FailureKind, FileOutcome, LineColumnSpan, ProjectRoot, + ReadFailure, StageFailure, }; use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; +use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; use sqlbuild_discovery::sql_tests::models::{ DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, }; +use sqlbuild_discovery::tree::main::display_text::display_text; use sqlbuild_discovery::tree::main::listings::read_listings; use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; +use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -29,11 +36,15 @@ use std::sync::Mutex; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use crate::bindings::models::{ModelDiscoveryRequest, NativeProjectTree, SqlTestDiscoveryRequest}; +use crate::bindings::models::{ + ModelDiscoveryRequest, ModelTextRequest, NativeProjectTree, SqlTestDiscoveryRequest, + SqlTestTextRequest, YamlDiscoveryRequest, +}; use crate::bindings::types::CompilerDetach; type PyObject = Py; type Located = (String, usize, usize, usize, usize); +type Listing = (PyObject, Vec<(PyObject, bool, bool)>); fn located(locations: Vec<(String, LineColumnSpan)>) -> Vec { locations @@ -50,6 +61,86 @@ fn object<'py, T: IntoPyObject<'py>>(py: Python<'py>, value: T) -> PyResult PyResult { + python_text(python_version, unicode_version).ok_or_else(|| { + PyValueError::new_err(format!( + "native discovery does not support Python {}.{} with Unicode {unicode_version}", + python_version.0, python_version.1 + )) + }) +} + +fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { + tuple_object( + py, + vec![ + object(py, "error")?, + object(py, failure.kind.as_str())?, + object(py, display_text(&failure.message))?, + object(py, failure.help)?, + ], + ) +} + +/// `("read", "os", errno, message, winerror)` or `("read", "decode", bytes, start, end, reason)`. +fn read_object(py: Python<'_>, failure: ReadFailure) -> PyResult { + match failure { + ReadFailure::Io { + errno, + winerror, + message, + } => tuple_object( + py, + vec![ + object(py, "read")?, + object(py, "os")?, + object(py, errno)?, + object(py, message)?, + object(py, winerror)?, + ], + ), + ReadFailure::Decode { + bytes, + start, + end, + reason, + } => tuple_object( + py, + vec![ + object(py, "read")?, + object(py, "decode")?, + PyBytes::new(py, &bytes).unbind().into_any(), + object(py, start)?, + object(py, end)?, + object(py, reason)?, + ], + ), + } +} + +/// A collection's failure payload; an unlistable directory is `("unlistable", path, read)`. +fn stage_failure_object( + py: Python<'_>, + tree: &ProjectTree, + failure: StageFailure, +) -> PyResult { + match failure { + StageFailure::Unlistable { + relative_path, + error, + } => tuple_object( + py, + vec![ + object(py, "unlistable")?, + path_object(py, tree, relative_path)?, + read_object(py, error)?, + ], + ), + StageFailure::Internal(reason) => Err(PyRuntimeError::new_err(reason)), + } +} + fn outcome_object( py: Python<'_>, outcome: FileOutcome, @@ -57,11 +148,33 @@ fn outcome_object( ) -> PyResult { match outcome { FileOutcome::Parsed(value) => parsed(py, value), - FileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), + FileOutcome::Unreadable(failure) => read_object(py, failure), FileOutcome::Failed(failure) => failure_object(py, failure), } } +/// Each discovered file as `(relative_path, payload)`, or the collection's failure payload. +fn files_object( + py: Python<'_>, + tree: &ProjectTree, + discovered: Result>, StageFailure>, + parsed: impl Fn(Python<'_>, T) -> PyResult, +) -> PyResult { + match discovered { + Ok(files) => files + .into_iter() + .map(|file| { + Ok(( + path_object(py, tree, file.relative_path)?, + outcome_object(py, file.outcome, &parsed)?, + )) + }) + .collect::>>()? + .into_py_any(py), + Err(failure) => stage_failure_object(py, tree, failure), + } +} + fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult { tuple_object( py, @@ -76,59 +189,93 @@ fn model_object(py: Python<'_>, model: DiscoveredModelFile) -> PyResult( - py: Python<'_>, - files: Vec>, - parsed: impl Fn(Python<'_>, T) -> PyResult, -) -> PyResult> { - files - .into_iter() - .map(|file| { - Ok(( - file.relative_path, - outcome_object(py, file.outcome, &parsed)?, - )) - }) - .collect() +/// A relative path as Python names it: the real, surrogate-escaped name where it is not UTF-8. +fn path_object(py: Python<'_>, tree: &ProjectTree, relative_path: String) -> PyResult { + match tree.raw_relative(&relative_path) { + Some(raw) => object(py, raw.into_os_string()), + None => object(py, relative_path), + } } -type Listing = (String, Vec<(String, bool, bool)>); -type FactRow = ( - String, - &'static str, - &'static str, - String, - Option, - String, -); -type DeclarationLayoutRows = (Option>, Option>); - -fn entry_rows(entries: &[TreeEntry]) -> Vec<(String, bool, bool)> { +fn entry_rows(py: Python<'_>, entries: &[TreeEntry]) -> PyResult> { entries .iter() - .map(|entry| (entry.name.clone(), entry.is_dir, entry.is_walkable_dir)) + .map(|entry| { + let name: PyObject = match &entry.raw_name { + Some(raw) => object(py, raw)?, + None => object(py, &entry.name)?, + }; + Ok((name, entry.is_dir, entry.is_walkable_dir)) + }) .collect() } -fn fact_row(fact: DeclarationFileFact) -> FactRow { - ( - fact.relative_path, - fact.kind.as_str(), - fact.scope_kind.as_str(), - fact.ownership_root, - fact.owning_path, - fact.declaration_root, +fn fact_row(py: Python<'_>, tree: &ProjectTree, fact: DeclarationFileFact) -> PyResult { + let owning_path: Option = fact + .owning_path + .map(|path| path_object(py, tree, path)) + .transpose()?; + tuple_object( + py, + vec![ + path_object(py, tree, fact.relative_path)?, + object(py, fact.kind.as_str())?, + object(py, fact.scope_kind.as_str())?, + path_object(py, tree, fact.ownership_root)?, + object(py, owning_path)?, + path_object(py, tree, fact.declaration_root)?, + ], ) } -fn group_row(group: DeclarationGroup) -> (String, String) { - (group.root, group.directory) +fn group_row(py: Python<'_>, tree: &ProjectTree, group: DeclarationGroup) -> PyResult { + tuple_object( + py, + vec![ + path_object(py, tree, group.root)?, + path_object(py, tree, group.directory)?, + ], + ) } -fn valid_rows(outcome: Result, DiscoveryFailure>, row: fn(T) -> R) -> Option> { +/// `("ok", rows)` for a valid layout, or the failure Python raises for it. +fn layout_rows( + py: Python<'_>, + tree: &ProjectTree, + outcome: Result, DiscoveryFailure>, + row: fn(Python<'_>, &ProjectTree, T) -> PyResult, +) -> PyResult { match outcome { - Ok(items) => Some(items.into_iter().map(row).collect()), - Err(_failure) => None, + Ok(items) => { + let rows: Vec = items + .into_iter() + .map(|item| row(py, tree, item)) + .collect::>()?; + tuple_object(py, vec![object(py, "ok")?, object(py, rows)?]) + } + Err(failure) => failure_object(py, failure), + } +} + +fn declaration_kind(kind: Option<&str>) -> PyResult> { + match kind { + None => Ok(None), + Some("macro") => Ok(Some(DeclarationKind::Macro)), + Some("enum") => Ok(Some(DeclarationKind::Enum)), + Some("constant") => Ok(Some(DeclarationKind::Constant)), + Some(other) => Err(PyValueError::new_err(format!( + "unknown declaration kind {other:?}" + ))), + } +} + +fn yaml_failure_kind(kind: &str) -> PyResult { + match kind { + "source" => Ok(FailureKind::Source), + "schema" => Ok(FailureKind::Schema), + other => Err(PyValueError::new_err(format!( + "unknown YAML file kind {other:?}" + ))), } } @@ -143,28 +290,31 @@ impl NativeProjectTree { } /// The listings read since the last call, for the pass's Python directory snapshot. - fn listings(&self) -> Vec { + fn listings(&self, py: Python<'_>) -> PyResult> { let Ok(mut exported) = self.exported.lock() else { - return Vec::new(); + return Ok(Vec::new()); }; read_listings(&self.inner) .into_iter() .filter(|(directory, _)| exported.insert(directory.clone())) - .map(|(directory, entries)| (directory, entry_rows(&entries))) + .map(|(directory, entries)| { + Ok(( + path_object(py, &self.inner, directory)?, + entry_rows(py, &entries)?, + )) + }) .collect() } } -/// Discover the model files natively, or return `None` when Python must run the stage. +/// Discover the model files: one payload per file, or the collection's failure payload. #[pyfunction] fn discover_model_files( py: Python<'_>, request: ModelDiscoveryRequest, tree: &NativeProjectTree, -) -> PyResult>> { - let Some(python) = python_text(request.python_version, &request.unicode_version) else { - return Ok(None); - }; +) -> PyResult { + let python: PythonText = python_semantics(request.python_version, &request.unicode_version)?; let root = ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -176,45 +326,59 @@ fn discover_model_files( extract_output_column_locations: request.extract_output_column_locations, python, }; - let discovered: Result>, StageDeferral> = py + let discovered = py .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match discovered { - Ok(files) => Ok(Some(files_object(py, files, model_object)?)), - Err(_deferral) => Ok(None), + files_object(py, &tree.inner, discovered, model_object) +} + +/// Parse in-memory SQL model contents into the payload model discovery returns for a file. +#[pyfunction] +fn parse_model_contents( + py: Python<'_>, + request: ModelTextRequest, + contents: String, +) -> PyResult { + let options = ModelFileOptions { + supported_keys: request.supported_keys, + removed_keys: request.removed_keys, + extract_implicit_alias_columns: request.extract_implicit_alias_columns, + extract_output_column_locations: request.extract_output_column_locations, + python: python_semantics(request.python_version, &request.unicode_version)?, + }; + let parsed = py + .compiler_detach(|| Ok(parse_model_text(&request.file_path, contents, &options))) + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; + match parsed { + Ok(model) => model_object(py, model), + Err(failure) => failure_object(py, failure), } } -/// The valid declaration file facts and named declaration groups; `None` where Python must scan. +/// The declaration file facts (of one kind, if given) and named groups, or the stage failure. #[pyfunction] +#[pyo3(signature = (tree, kind=None))] fn discover_declaration_layout( py: Python<'_>, tree: &NativeProjectTree, -) -> PyResult> { - let layout: Result = py - .compiler_detach(|| Ok(declaration_layout(&tree.inner))) + kind: Option<&str>, +) -> PyResult { + let kind: Option = declaration_kind(kind)?; + let layout: Result = py + .compiler_detach(|| Ok(declaration_layout(&tree.inner, kind))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; match layout { - Ok(layout) => Ok(Some(( - valid_rows(layout.file_facts, fact_row), - valid_rows(layout.named_groups, group_row), - ))), - Err(_deferral) => Ok(None), + Ok(layout) => tuple_object( + py, + vec![ + layout_rows(py, &tree.inner, layout.file_facts, fact_row)?, + layout_rows(py, &tree.inner, layout.named_groups, group_row)?, + ], + ), + Err(failure) => stage_failure_object(py, &tree.inner, failure), } } -fn failure_object(py: Python<'_>, failure: DiscoveryFailure) -> PyResult { - tuple_object( - py, - vec![ - object(py, "error")?, - object(py, failure.kind.as_str())?, - object(py, failure.message)?, - object(py, failure.help)?, - ], - ) -} - fn test_block_object(py: Python<'_>, block: SqlTestBlock) -> PyResult { tuple_object( py, @@ -258,10 +422,12 @@ fn scenario_object(py: Python<'_>, file: DiscoveredScenarioFile) -> PyResult Option<(ProjectRoot, SqlTestFileOptions)> { - let python: PythonText = python_text(request.python_version, &request.unicode_version)?; - Some(( +/// The root and options of one SQL test or scenario discovery request. +fn sql_test_inputs( + request: SqlTestDiscoveryRequest, +) -> PyResult<(ProjectRoot, SqlTestFileOptions)> { + let python: PythonText = python_semantics(request.python_version, &request.unicode_version)?; + Ok(( ProjectRoot { directory: PathBuf::from(request.project_dir), display_prefix: request.display_prefix, @@ -274,78 +440,143 @@ fn sql_test_inputs(request: SqlTestDiscoveryRequest) -> Option<(ProjectRoot, Sql )) } -/// Discover and split the SQL test files natively, or return `None` when Python must run. +/// Discover and split the SQL test files: one payload per file, or the failure payload. #[pyfunction] fn discover_sql_test_files( py: Python<'_>, request: SqlTestDiscoveryRequest, tree: &NativeProjectTree, -) -> PyResult>> { - let Some((root, options)) = sql_test_inputs(request) else { - return Ok(None); - }; - let discovered: Result>, StageDeferral> = py +) -> PyResult { + let (root, options) = sql_test_inputs(request)?; + let discovered = py .compiler_detach(|| Ok(discover_tests(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match discovered { - Ok(files) => Ok(Some(files_object(py, files, test_file_object)?)), - Err(_deferral) => Ok(None), - } + files_object(py, &tree.inner, discovered, test_file_object) } -/// Discover and header-parse the scenario files natively, or return `None` when Python must run. +/// Discover and header-parse the scenario files: one payload per file, or the failure payload. #[pyfunction] fn discover_scenario_files( py: Python<'_>, request: SqlTestDiscoveryRequest, tree: &NativeProjectTree, -) -> PyResult>> { - let Some((root, options)) = sql_test_inputs(request) else { - return Ok(None); - }; - let discovered: Result>, StageDeferral> = py +) -> PyResult { + let (root, options) = sql_test_inputs(request)?; + let discovered = py .compiler_detach(|| Ok(discover_scenarios(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match discovered { - Ok(files) => Ok(Some(files_object(py, files, scenario_object)?)), - Err(_deferral) => Ok(None), + files_object(py, &tree.inner, discovered, scenario_object) +} + +/// The options of one in-memory SQL test or scenario request. +fn text_options(request: &SqlTestTextRequest) -> PyResult { + Ok(SqlTestFileOptions { + test_keys: request.test_keys.clone(), + scenario_keys: request.scenario_keys.clone(), + python: python_semantics(request.python_version, &request.unicode_version)?, + }) +} + +/// Split in-memory SQL test contents into the payload test discovery returns for a file. +#[pyfunction] +fn parse_sql_test_contents( + py: Python<'_>, + request: SqlTestTextRequest, + contents: String, +) -> PyResult { + let options: SqlTestFileOptions = text_options(&request)?; + let parsed = py + .compiler_detach(|| Ok(parse_sql_test_text(&request.file_path, contents, &options))) + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; + match parsed { + Ok(file) => test_file_object(py, file), + Err(failure) => failure_object(py, failure), } } -/// One file's payload; a value Python cannot hold natively makes Python load that file. -fn yaml_file_object(py: Python<'_>, outcome: YamlFileOutcome) -> PyResult { - match outcome { - YamlFileOutcome::Loaded { contents, value } => match config_value_to_python(py, value) { - Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), - Err(_unconvertible) => { - tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) - } - }, - YamlFileOutcome::LoadInPython { contents } => { - tuple_object(py, vec![object(py, "load")?, object(py, contents)?]) - } - YamlFileOutcome::Unreadable => tuple_object(py, vec![object(py, "read")?]), +/// Header-parse in-memory SQL scenario contents into the payload scenario discovery returns. +#[pyfunction] +fn parse_scenario_contents( + py: Python<'_>, + request: SqlTestTextRequest, + contents: String, +) -> PyResult { + let options: SqlTestFileOptions = text_options(&request)?; + let parsed = py + .compiler_detach(|| Ok(parse_scenario_text(&request.file_path, contents, &options))) + .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; + match parsed { + Ok(file) => scenario_object(py, file), + Err(failure) => failure_object(py, failure), } } -/// Read and load YAML files natively, one payload per path, or `None` when Python must run. +/// `("ok", contents, value)`; a value Python cannot build fails like invalid YAML. +fn loaded_yaml_object( + py: Python<'_>, + (file_path, kind): (&str, FailureKind), + (contents, value): (String, sqlbuild_config::models::ConfigValue), +) -> PyResult { + match config_value_to_python(py, value) { + Ok(loaded) => tuple_object(py, vec![object(py, "ok")?, object(py, contents)?, loaded]), + Err(error) => failure_object( + py, + DiscoveryFailure::new( + kind, + format!("{file_path} contains invalid YAML: {}", error.value(py)), + ), + ), + } +} + +/// Read and load YAML files: one payload per path, or the collection's failure payload. #[pyfunction] fn load_yaml_files( py: Python<'_>, + request: YamlDiscoveryRequest, relative_paths: Vec, tree: &NativeProjectTree, -) -> PyResult>> { - let loaded: Result, StageDeferral> = py - .compiler_detach(|| Ok(load_yaml(&tree.inner, &relative_paths))) +) -> PyResult { + let kind: FailureKind = yaml_failure_kind(&request.kind)?; + let root = ProjectRoot { + directory: PathBuf::from(request.project_dir), + display_prefix: request.display_prefix, + }; + let loaded: Result, StageFailure> = py + .compiler_detach(|| Ok(load_yaml(&root, &tree.inner, &relative_paths, kind))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - match loaded { - Ok(outcomes) => Ok(Some( - outcomes - .into_iter() - .map(|outcome| yaml_file_object(py, outcome)) - .collect::>()?, - )), - Err(_deferral) => Ok(None), + let outcomes: Vec = match loaded { + Ok(outcomes) => outcomes, + Err(failure) => return stage_failure_object(py, &tree.inner, failure), + }; + outcomes + .into_iter() + .zip(&relative_paths) + .map(|(outcome, relative_path)| match outcome { + YamlFileOutcome::Loaded { contents, value } => loaded_yaml_object( + py, + (&root.display_path(relative_path), kind), + (contents, value), + ), + YamlFileOutcome::Failed(failure) => failure_object(py, failure), + YamlFileOutcome::Unreadable(failure) => read_object(py, failure), + }) + .collect::>>()? + .into_py_any(py) +} + +/// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. +#[pyfunction] +fn load_yaml_document( + py: Python<'_>, + file_path: &str, + text: String, + kind: &str, +) -> PyResult { + let kind: FailureKind = yaml_failure_kind(kind)?; + match load_yaml_text(file_path, &text, kind) { + Ok(value) => loaded_yaml_object(py, (file_path, kind), (text, value)), + Err(failure) => failure_object(py, failure), } } @@ -358,10 +589,14 @@ fn native_text_supported(python_version: (u8, u8), unicode_version: &str) -> boo pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(native_text_supported, module)?)?; module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_document, module)?)?; module.add_function(wrap_pyfunction!(discover_sql_test_files, module)?)?; module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_test_contents, module)?)?; + module.add_function(wrap_pyfunction!(parse_scenario_contents, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index cc5d970dbe..5e08cd1970 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -131,3 +131,36 @@ pub(crate) struct SqlTestDiscoveryRequest { pub(crate) python_version: (u8, u8), pub(crate) unicode_version: String, } + +/// One in-memory SQL test or scenario parse request; `file_path` is the path messages name. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct SqlTestTextRequest { + pub(crate) file_path: String, + pub(crate) test_keys: Vec, + pub(crate) scenario_keys: Vec, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, +} + +/// One native YAML file discovery request; `kind` is `source` or `schema`. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct YamlDiscoveryRequest { + pub(crate) project_dir: String, + pub(crate) display_prefix: String, + pub(crate) kind: String, +} + +/// One in-memory SQL model parse request; `file_path` is the path messages name. +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct ModelTextRequest { + pub(crate) file_path: String, + pub(crate) supported_keys: Vec, + pub(crate) removed_keys: Vec, + pub(crate) extract_implicit_alias_columns: bool, + pub(crate) extract_output_column_locations: bool, + pub(crate) python_version: (u8, u8), + pub(crate) unicode_version: String, +} diff --git a/pyproject.toml b/pyproject.toml index 30d28b40a0..800216629c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ name = "sqlbuild" version = "0.139.0" description = "Typed, test-first SQL pipelines with local E2E testing" readme = "README.md" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.15" dependencies = [ "duckdb>=1.3.2", "filelock>=3.20.3", diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index 09701b1305..6c9d0b5217 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -1,5 +1,7 @@ """Minimal projects that fail inside project discovery, one per diagnostic family.""" +import os + from scripts.compiler_differential._helpers.corpus.case_builder import ( config_files, failure_case, @@ -101,6 +103,12 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="model-path-not-utf8", + expected_code="D016", + files={os.fsdecode(b"models/staging/caf\xe9.sql"): FAILURE_BASE_STAGING}, + expected_message="is not valid UTF-8; rename it so SQLBuild can read it", + ), failure_case( name="model-reserved-name", expected_code="D016", @@ -216,6 +224,21 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: "seeds/order_channels.csv": "id,label\n1,web\n", }, ), + failure_case( + name="schema-yml-invalid-yaml", + expected_code="D005", + files={"models/marts/schema.yml": "models: [\n"}, + expected_message="contains invalid YAML at line 2, column 1", + ), + failure_case( + name="source-unsupported-yaml", + expected_code="D006", + files={ + FAILURE_SOURCES_PATH: FAILURE_BASE_FILES[FAILURE_SOURCES_PATH] + + " meta:\n blob: !!binary aGk=\n" + }, + expected_message="uses YAML that SQLBuild does not support at line", + ), failure_case( name="source-unknown-key", expected_code="D006", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 05f3f8cdac..e9cad59362 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -473,6 +473,9 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "compile raises the strict scope index error, which reports it inside a P001 " "'Invalid declaration scope index' message" ) +DISCOVERY_UNREACHABLE_CODES: dict[str, str] = { + "D017": ("the corpus runs a supported Python; an integration test forces an unsupported one"), +} RENDER_UNREACHABLE_CODES: dict[str, str] = { "S001": _SCOPE_QUERY_ONLY, "S002": _SCOPE_QUERY_ONLY, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md index 1cd032f86d..8e15f0e76f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/resource-identities.md @@ -40,6 +40,13 @@ error[D016]: Invalid model identity 'DailyOrders' in models/DailyOrders.sql; use snake_case 'daily_orders' ``` +Discovery also fails with `D016` when the project directory, or a model, unit test, scenario, source +or schema YAML file it reads, has a path that is not valid UTF-8, including a directory above the +file. The error shows each invalid byte as a `\xNN` escape; rename the path to valid UTF-8. Other +files with such names, such as notes or data files, and directories that hold none of those files +are ignored. Commands that tolerate unreadable files, such as `sqb scope`, report the file and +continue. + SQLBuild suggests a corrected spelling but never silently normalizes an identity. Silent normalization would make selectors, manifests, persisted execution state, and integration keys disagree about which resource ran. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md index 5e21ef3e48..c3f779ec52 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/sources/ - Managed sources (loaders) - Source freshness - Config reference +- YAML that SQLBuild reads Sources declare external data that your models depend on. They are defined in YAML files under `sources/` in your project directory. @@ -297,3 +298,25 @@ Use [`sqb freshness`](../cli/freshness.md) to observe source freshness on demand A sources file contains only `sources` and `dlt_sources`. Unknown keys in a sources file, a source, a source column, `freshness`, `freshness.age_policy`, or `ingestr` fail discovery with `D006`, naming the key and suggesting the nearest supported key. + +## YAML that SQLBuild reads + +Sources files, seed declarations and `schema.yml` files use plain YAML 1.1: block and flow mappings +and sequences, plain and quoted scalars, literal (`|`) and folded (`>`) block scalars, comments, +anchors, aliases and merge keys (`<<`). Files must be UTF-8. + +Invalid YAML fails discovery with `D006` (sources) or `D005` (seed declarations and `schema.yml`), +naming the file, line and column. The following YAML features also fail with the same codes and a +hint to rewrite the value in plain YAML: + +- tab characters anywhere in the file, and byte order marks after its start +- `%YAML` and `%TAG` directives +- the `!!binary`, `!!set`, `!!omap` and `!!pairs` tags, and an empty scalar tagged `!` +- recursive aliases, and aliases that expand beyond SQLBuild's size limit +- nesting deeper than 256 levels, integers longer than 4300 digits, and implicit keys longer than + 1000 characters +- NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR line breaks +- `?` inside a plain scalar in a flow collection, and `:` directly before `,`, `]` or `}` in a + flow collection +- a block scalar used as a key, inside a flow collection, or with indentation SQLBuild cannot read + unambiguously diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md index 77c7a7a981..6c5d5d733a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md @@ -21,7 +21,7 @@ This guide walks you through creating and running a complete transformation proj ## Prerequisites -- Python 3.12+ +- Python 3.12–3.14 (other versions stop with `D017` naming the supported versions) - SQLBuild installed: `uv pip install sqlbuild` or `pip install sqlbuild` Confirm the installed release with `sqb --version`. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index acb25ca011..4aa1d15abf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -51,23 +51,23 @@ class NativeProjectTree: def discover_model_files( request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | None: ... +) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... def discover_sql_test_files( request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | None: ... +) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... def discover_scenario_files( request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | None: ... -def load_yaml_files(relative_paths: list[str], tree: NativeProjectTree) -> list[object] | None: ... +) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... +def load_yaml_files( + request: dict[str, object], relative_paths: list[str], tree: NativeProjectTree +) -> list[tuple[object, ...]] | tuple[object, ...]: ... +def load_yaml_document(file_path: str, text: str, kind: str) -> tuple[object, ...]: ... def discover_declaration_layout( - tree: NativeProjectTree, -) -> ( - tuple[ - list[tuple[str, str, str, str, str | None, str]] | None, - list[tuple[str, str]] | None, - ] - | None -): ... + tree: NativeProjectTree, kind: str | None = None +) -> tuple[tuple[object, ...], tuple[object, ...]] | tuple[object, ...]: ... +def parse_model_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... +def parse_sql_test_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... +def parse_scenario_contents(request: dict[str, object], contents: str) -> tuple[object, ...]: ... class NativeScopeIndex: def records( diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 06a5f6136e..1db8f83d16 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -45,7 +45,7 @@ from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryCacheRequest +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.main.selected_graph import ( @@ -122,9 +122,6 @@ def _analyze_compile_project( sql_analysis_enabled_override=( False if profile_flags.skip_discovery_sql_analysis else None ), - cache_request=DiscoveryCacheRequest( - selected_target=selected_target, no_cache=analysis_selection.no_cache - ), declaration_reuse=COMPILE_RENDER_REUSE.claim_discovery(), ) reject_unit_test_selectors(select=select, exclude=exclude, discovered_inputs=discovered_inputs) diff --git a/src/sqlbuild/cli/entry/_helpers/__init__.py b/src/sqlbuild/cli/entry/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/entry/_helpers/streams.py b/src/sqlbuild/cli/entry/_helpers/streams.py new file mode 100644 index 0000000000..eee08dc4b8 --- /dev/null +++ b/src/sqlbuild/cli/entry/_helpers/streams.py @@ -0,0 +1,24 @@ +"""Standard output streams the CLI can always write, whatever the console code page.""" + +import codecs +import io +import sys + +from sqlbuild.cli.entry.constants import OUTPUT_STREAM_ENCODING, OUTPUT_STREAM_ERRORS + + +def use_utf8_output_streams() -> tuple[bool, ...]: + """Write stdout and stderr as UTF-8 when opened with another encoding; return which changed.""" + + return tuple(use_utf8_stream(stream) for stream in (sys.stdout, sys.stderr)) + + +def use_utf8_stream(stream: object) -> bool: + """Reconfigure a text stream that is not UTF-8 to UTF-8; return whether it changed.""" + + if not isinstance(stream, io.TextIOWrapper): + return False + if codecs.lookup(stream.encoding).name == codecs.lookup(OUTPUT_STREAM_ENCODING).name: + return False + stream.reconfigure(encoding=OUTPUT_STREAM_ENCODING, errors=OUTPUT_STREAM_ERRORS) + return True diff --git a/src/sqlbuild/cli/entry/constants.py b/src/sqlbuild/cli/entry/constants.py index 6199328032..3bff31b705 100644 --- a/src/sqlbuild/cli/entry/constants.py +++ b/src/sqlbuild/cli/entry/constants.py @@ -2,3 +2,5 @@ NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE: str = "MIMALLOC_PURGE_DELAY" NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS: str = "10" +OUTPUT_STREAM_ENCODING: str = "utf-8" +OUTPUT_STREAM_ERRORS: str = "backslashreplace" diff --git a/src/sqlbuild/cli/entry/main/entry.py b/src/sqlbuild/cli/entry/main/entry.py index 4e938aaae3..f89001d0e5 100644 --- a/src/sqlbuild/cli/entry/main/entry.py +++ b/src/sqlbuild/cli/entry/main/entry.py @@ -3,6 +3,7 @@ import os from collections.abc import Sequence +from sqlbuild.cli.entry._helpers.streams import use_utf8_output_streams from sqlbuild.cli.entry.constants import ( NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, @@ -15,6 +16,7 @@ def main(argv: Sequence[str] | None = None) -> int: os.environ.setdefault( NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS ) + _ = use_utf8_output_streams() from sqlbuild.cli.commands.main.entrypoint.entry import main as run_cli return run_cli(argv) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index c8e4494128..0ee7f44a1e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -18,28 +18,21 @@ discover_hook_functions, discover_macro_files, discover_materialization_files, - discover_model_files, discover_model_schema_files, discover_provider_classes, discover_python_function_files, discover_python_node_functions, - discover_scenario_files, - discover_schema_files, discover_seed_files, - discover_source_files, discover_sql_function_files, discover_sql_hook_files, - discover_test_files, ) from sqlbuild.compiler.discovery._helpers.integrations.loaders import ( build_integration_loader_functions, ) -from sqlbuild.compiler.discovery._helpers.native.declarations import ( - prepare_native_declaration_layout, -) from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) +from sqlbuild.compiler.discovery._helpers.native.payloads import native_text_runtime from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( discover_native_scenario_files, discover_native_test_files, @@ -75,7 +68,6 @@ TolerantScopeDiscovery, ) from sqlbuild.compiler.discovery.types import DeclarationFilesReuse -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -87,9 +79,7 @@ def build_discovered_project_inputs( local_config: LocalConfig, sql_analysis_enabled: bool, extract_output_column_locations: bool = True, - fact_cache: FactCacheStore | None = None, declaration_reuse: DeclarationFilesReuse | None = None, - native: bool = False, ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" @@ -101,14 +91,11 @@ def build_discovered_project_inputs( project_dir=project_dir, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, - native=native, ) discover: Callable[[], DiscoveredDeclarationFiles] = partial( _discover_declaration_files, project_dir=project_dir, discover_models=discover_models, - fact_cache=fact_cache, - native=native, ) declarations: DiscoveredDeclarationFiles = ( discover() @@ -226,18 +213,12 @@ def _discover_declaration_files( *, project_dir: Path, discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], - fact_cache: FactCacheStore | None, - native: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): - source_files: tuple[DiscoveredSourceFile, ...] = ( - discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache) - if native - else discover_source_files(project_dir=project_dir, fact_cache=fact_cache) + source_files: tuple[DiscoveredSourceFile, ...] = discover_native_source_files( + project_dir=project_dir ) model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() - if native: - prepare_native_declaration_layout(project_dir=project_dir) return DiscoveredDeclarationFiles( source_files=source_files, model_files=model_files, @@ -247,22 +228,10 @@ def _discover_declaration_files( sql_function_files=discover_sql_function_files(project_dir=project_dir), sql_hook_files=discover_sql_hook_files(project_dir=project_dir), python_function_files=discover_python_function_files(project_dir=project_dir), - schema_files=( - discover_native_schema_files(project_dir=project_dir) - if native - else discover_schema_files(project_dir=project_dir) - ), + schema_files=discover_native_schema_files(project_dir=project_dir), seed_files=discover_seed_files(project_dir=project_dir), - test_files=( - discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache) - if native - else discover_test_files(project_dir=project_dir, fact_cache=fact_cache) - ), - scenario_files=( - discover_native_scenario_files(project_dir=project_dir) - if native - else discover_scenario_files(project_dir=project_dir) - ), + test_files=discover_native_test_files(project_dir=project_dir), + scenario_files=discover_native_scenario_files(project_dir=project_dir), audit_files=discover_audit_files(project_dir=project_dir), macro_files=discover_macro_files(project_dir=project_dir), adapter_file=discover_adapter_file(project_dir=project_dir), @@ -274,17 +243,12 @@ def _discover_model_files( project_dir: Path, sql_analysis_enabled: bool, extract_output_column_locations: bool, - native: bool, ) -> tuple[DiscoveredSqlModelFile, ...]: - discover: Callable[..., tuple[DiscoveredSqlModelFile, ...]] = ( - discover_native_model_files if native else discover_model_files + return discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=sql_analysis_enabled, + extract_output_column_locations=extract_output_column_locations, ) - with DirectorySnapshot.scope(project_dir=project_dir): - return discover( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: @@ -295,6 +259,7 @@ def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscove def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: + _ = native_text_runtime() project_config, local_config, config_faults = _discover_configs(project_dir=project_dir) models, model_faults = _discover_models(project_dir=project_dir) enums, constants, macros, declaration_faults = _discover_declarations(project_dir=project_dir) @@ -306,7 +271,7 @@ def _build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscov function=discover_sql_hook_files, project_dir=project_dir ) sources, source_faults = _discover_category( - function=discover_source_files, project_dir=project_dir + function=discover_native_source_files, project_dir=project_dir ) audits, audit_faults = _discover_category( function=discover_audit_files, project_dir=project_dir @@ -376,7 +341,7 @@ def _discover_models( *, project_dir: Path ) -> tuple[tuple[DiscoveredSqlModelFile, ...], tuple[DiscoveryFileFault, ...]]: faults: list[DiscoveryFileFault] = [] - models: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( + models: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( project_dir=project_dir, extract_implicit_alias_columns=False, extract_output_column_locations=False, @@ -438,10 +403,10 @@ def _discover_relationships( tuple[DiscoveryFileFault, ...], ]: faults: list[DiscoveryFileFault] = [] - tests: tuple[DiscoveredSqlTestFile, ...] = discover_test_files( + tests: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( project_dir=project_dir, on_fault=faults.append ) - scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_scenario_files( + scenarios: tuple[DiscoveredSqlScenarioFile, ...] = discover_native_scenario_files( project_dir=project_dir, on_fault=faults.append ) return tests, scenarios, tuple(faults) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/cache_root.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/cache_root.py deleted file mode 100644 index b177f131b8..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/cache_root.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Compile-cache root resolution for discovery facts.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveryCacheRequest -from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig - - -def discovery_cache_root( - *, - project_dir: Path, - project_config: ProjectConfig, - local_config: LocalConfig, - cache_request: DiscoveryCacheRequest | None, -) -> Path | None: - """Return the compile-cache root for discovery facts, or None when caching is off.""" - - if cache_request is None or cache_request.no_cache: - return None - try: - target_name: str | None = resolve_target_name( - project_config=project_config, - local_config=local_config, - selected_target=cache_request.selected_target, - ) - target_config: TargetConfig | None = ( - resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) - if target_name is not None - else None - ) - except SpecConfigError: - return None - return compile_cache_root( - project_dir=project_dir, target_config=target_config, no_cache=cache_request.no_cache - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/cached_files.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/cached_files.py deleted file mode 100644 index 1b363b92ab..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/cached_files.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Compact fact-cache codecs for discovered files whose raw contents are re-read live.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_sources_yml -from sqlbuild.compiler.discovery.constants import DISCOVERY_SOURCE_FACT_KIND -from sqlbuild.compiler.discovery.models import ( - DiscoveredSourceFile, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore - - -def parse_source_file_with_cache( - *, - project_dir: Path, - file_path: Path, - relative_path: Path, - fact_cache: FactCacheStore | None, -) -> DiscoveredSourceFile: - """Parse one source YAML file, reusing its exact cached fact when the contents match.""" - - contents: str = file_path.read_text(encoding="utf-8") - cache_key: str | None = ( - fact_cache.key(DISCOVERY_SOURCE_FACT_KIND, str(project_dir), str(file_path), contents) - if fact_cache is not None and fact_cache.enabled - else None - ) - slot: str = f"{DISCOVERY_SOURCE_FACT_KIND}:{file_path}" - if fact_cache is not None and cache_key is not None: - cached: DiscoveredSourceFile | None = decode_cached_source_file( - fact=fact_cache.read_many(((slot, cache_key),)).get(cache_key), - file_path=file_path, - contents=contents, - ) - if cached is not None: - return cached - source_file: DiscoveredSourceFile = DiscoveredSourceFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - source_entries=parse_sources_yml(contents=contents, file_path=file_path), - ) - if fact_cache is not None and cache_key is not None: - fact_cache.stage(key=cache_key, slot=slot, value=encode_cached_source_file(source_file)) - return source_file - - -def encode_cached_source_file(source_file: DiscoveredSourceFile) -> object: - """Return a source fact without raw YAML.""" - - return replace(source_file, contents="") - - -def decode_cached_source_file( - *, fact: object, file_path: Path, contents: str -) -> DiscoveredSourceFile | None: - """Rebuild one cached source file from its fact and exact live YAML.""" - - if not isinstance(fact, DiscoveredSourceFile) or fact.file_path != file_path: - return None - return replace(fact, contents=contents) - - -def encode_cached_sql_test_file(test_file: DiscoveredSqlTestFile) -> object: - """Return a SQL test fact without raw file contents.""" - - return replace(test_file, contents="") - - -def decode_cached_sql_test_file( - *, fact: object, file_path: Path, contents: str -) -> DiscoveredSqlTestFile | None: - """Rebuild one cached SQL test file from its fact and exact live contents.""" - - if not isinstance(fact, DiscoveredSqlTestFile) or fact.file_path != file_path: - return None - return replace(fact, contents=contents) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 6b74f2b4b1..1d0ab8a1aa 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,12 +14,6 @@ from typing import TYPE_CHECKING, cast, get_type_hints from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( - parse_source_file_with_cache, -) -from sqlbuild.compiler.discovery._helpers.filesystem.model_files import ( - discover_matched_model_file, -) from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( named_declaration_files, named_declaration_roots, @@ -28,7 +22,7 @@ is_in_scoped_declaration_tree, project_relative_path, ) -from sqlbuild.compiler.discovery._helpers.filesystem.sql_test_files import discover_sql_test_files +from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_file_facts from sqlbuild.compiler.discovery._helpers.python.functions import parse_python_function from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file from sqlbuild.compiler.discovery._helpers.sql.declarations import ( @@ -38,27 +32,14 @@ ) from sqlbuild.compiler.discovery._helpers.sql.functions import parse_function_sql from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - match_model_headers, - prepare_matched_model_file_headers, -) -from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file -from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( - schema_file_paths, - source_file_paths, -) -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( - CANONICAL_AUTHORED_ROOTS, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SEED_FILE_SUFFIX, ) from sqlbuild.compiler.discovery.exceptions import ( - DeclarationParseError, EventExporterDiscoveryError, - ModelSqlParseError, ProviderDiscoveryError, PythonNodeDiscoveryError, ) @@ -82,32 +63,17 @@ DiscoveredProviderUsage, DiscoveredPythonFunctionFile, DiscoveredPythonNodeFunctions, - DiscoveredSchemaFile, DiscoveredSeedFile, - DiscoveredSourceFile, DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, DiscoveredTaskFunction, DiscoveryFileFault, - ModelHeaderMatch, NamedDeclarationRoot, ) -from sqlbuild.compiler.discovery.types import NativeDeclarationFact, ScopedDeclarationFile -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.discovery.types import ScopedDeclarationFile from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_DIRECTORY_FACTS, - DECLARATION_GROUP_DIRECTORY, - GLOBAL_DECLARATION_DIRECTORIES, - GROUPED_NAMED_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, - SCOPED_DECLARATION_DIRECTORIES, -) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.provider.exceptions import ProviderInputError from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition @@ -135,7 +101,6 @@ ) from sqlbuild.runtime.event_exporting.models import LifecycleEventSinkDefinition from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry if TYPE_CHECKING: from pydantic import ValidationError @@ -189,19 +154,7 @@ def _discover_declaration_file_facts( cached: object = tree.memo.get(memo_key) if cached is not None: return cast(tuple[_DeclarationFileFacts, ...], cached) - facts: tuple[_DeclarationFileFacts, ...] = _scan_declaration_file_facts( - project_dir=project_dir, declaration_kind=declaration_kind, tree=tree - ) - tree.memo[memo_key] = facts - return facts - - -def remember_declaration_file_facts( - *, project_dir: Path, facts: Iterable[NativeDeclarationFact] -) -> None: - """Record a scan of every declaration kind another walk of this pass already validated.""" - - DirectorySnapshot.current(project_dir=project_dir).memo[(_FACTS_MEMO_KEY, None)] = tuple( + facts: tuple[_DeclarationFileFacts, ...] = tuple( _DeclarationFileFacts( file_path=project_dir / relative_path, relative_path=Path(relative_path), @@ -211,235 +164,14 @@ def remember_declaration_file_facts( owning_path=None if owning_path is None else Path(owning_path), declaration_root=Path(declaration_root), ) - for relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root in facts - ) - - -def _scan_declaration_file_facts( - *, project_dir: Path, declaration_kind: DeclarationKind | None, tree: DirectorySnapshot -) -> tuple[_DeclarationFileFacts, ...]: - _validate_declaration_groups(project_dir=project_dir, tree=tree) - for directory_name in sorted(LOCAL_DECLARATION_DIRECTORIES): - directory_kind, _scope_kind = DECLARATION_DIRECTORY_FACTS[directory_name] - if (declaration_kind is None or directory_kind is declaration_kind) and ( - project_dir / directory_name - ).is_dir(): - raise DeclarationParseError( - f"Scoped declaration root {directory_name}/ must be below a canonical authored root" - ) - - facts: list[_DeclarationFileFacts] = [] - global_directory: str - for global_directory in sorted(GLOBAL_DECLARATION_DIRECTORIES): - directory_kind, _scope_kind = DECLARATION_DIRECTORY_FACTS[global_directory] - if declaration_kind is not None and directory_kind is not declaration_kind: - continue - declaration_root: Path = project_dir / global_directory - if declaration_root.is_dir(): - facts.extend( - _declaration_files_under_root( - project_dir=project_dir, - tree=tree, - ownership_root=Path(global_directory), - declaration_root=declaration_root, - owning_path=None, - scope_kind=ScopeKind.GLOBAL, - ) - ) - - root_components: tuple[str, ...] - for root_components in CANONICAL_AUTHORED_ROOTS: - authored_root: Path = project_dir.joinpath(*root_components) - if not authored_root.is_dir(): - continue - directory: Path - for directory in sorted(tree.directories(root=authored_root)): - if directory.name not in SCOPED_DECLARATION_DIRECTORIES: - continue - directory_kind, directory_scope_kind = DECLARATION_DIRECTORY_FACTS[directory.name] - if declaration_kind is not None and directory_kind is not declaration_kind: - continue - relative_directory: Path = project_relative_path( - path=directory, project_dir=project_dir - ) - descendants: tuple[str, ...] = relative_directory.parts[len(root_components) :] - if any(part in SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): - raise DeclarationParseError( - f"Declaration root {relative_directory.as_posix()}/ is nested inside another " - "declaration tree" - ) - owning_path: Path = relative_directory.parent - if relative_directory.parent.name == DECLARATION_GROUP_DIRECTORY: - owning_path = relative_directory.parent.parent - facts.extend( - _declaration_files_under_root( - project_dir=project_dir, - tree=tree, - ownership_root=Path(*root_components), - declaration_root=directory, - owning_path=owning_path, - scope_kind=directory_scope_kind, - ) - ) - return tuple(sorted(facts, key=lambda item: item.relative_path.as_posix())) - - -def _validate_declaration_groups(*, project_dir: Path, tree: DirectorySnapshot) -> None: - root_group: Path = project_dir / DECLARATION_GROUP_DIRECTORY - if root_group.exists(): - raise DeclarationParseError( - f"Grouped declaration root {DECLARATION_GROUP_DIRECTORY}/ must be below a canonical " - "authored root" - ) - for root_components in CANONICAL_AUTHORED_ROOTS: - authored_root: Path = project_dir.joinpath(*root_components) - if not authored_root.is_dir(): - continue - for group in sorted( - path - for path in tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY) - if path.is_dir() - ): - if group.parent == authored_root: - raise DeclarationParseError( - f"Grouped declaration root {group.relative_to(project_dir).as_posix()}/ must " - "be below a concrete owner directory; use the project-wide macros/, enums/, " - "constants/, audits/, schemas/, or hooks/ root instead" - ) - unsupported: tuple[Path, ...] = tuple( - sorted( - child - for child in group.iterdir() - if not child.is_dir() - or child.name - not in SCOPED_DECLARATION_DIRECTORIES | GROUPED_NAMED_DECLARATION_DIRECTORIES - ) + for relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root in ( + native_declaration_file_facts( + project_dir=project_dir, declaration_kind=declaration_kind ) - if unsupported: - rendered: str = ", ".join( - project_relative_path(path=path, project_dir=project_dir).as_posix() - for path in unsupported - ) - raise DeclarationParseError( - f"Declaration group {group.relative_to(project_dir).as_posix()}/ contains " - f"unsupported entries: {rendered}" - ) - - -def _declaration_files_under_root( - *, - project_dir: Path, - tree: DirectorySnapshot, - ownership_root: Path, - declaration_root: Path, - owning_path: Path | None, - scope_kind: ScopeKind, -) -> list[_DeclarationFileFacts]: - nested_root: Path - for nested_root in sorted( - path - for path in tree.directories(root=declaration_root) - if path.name in SCOPED_DECLARATION_DIRECTORIES - ): - relative_nested_root: str = project_relative_path( - path=nested_root, project_dir=project_dir - ).as_posix() - raise DeclarationParseError( - f"Declaration root {relative_nested_root}/ is nested inside another declaration tree" ) - declaration_kind: DeclarationKind = DECLARATION_DIRECTORY_FACTS[declaration_root.name][0] - suffix: str = ".py" if declaration_kind is DeclarationKind.MACRO else ".sql" - results: list[_DeclarationFileFacts] = [] - file_path: Path - for file_path in sorted(tree.rglob(root=declaration_root, pattern=f"*{suffix}")): - if declaration_kind is DeclarationKind.MACRO and file_path.stem == PYTHON_INIT_MODULE_STEM: - continue - results.append( - _DeclarationFileFacts( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - declaration_kind=declaration_kind, - scope_kind=scope_kind, - ownership_root=ownership_root, - owning_path=owning_path, - declaration_root=project_relative_path( - path=declaration_root, project_dir=project_dir - ), - ) - ) - return results - - -def discover_model_files( - *, - project_dir: Path, - extract_implicit_alias_columns: bool = True, - extract_output_column_locations: bool = True, - selected_model_names: frozenset[str] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, -) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover SQL model files under models/.""" - - model_root: Path = project_dir / "models" - if not model_root.is_dir(): - return () - - loaded_model_files: list[tuple[Path, str | None, Exception | None]] = [] - file_path: Path - for file_path in _sorted_glob(project_dir=project_dir, root=model_root, pattern="*.sql"): - if selected_model_names is not None and file_path.stem not in selected_model_names: - continue - if is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir): - continue - try: - loaded_model_files.append((file_path, file_path.read_text(encoding="utf-8"), None)) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - loaded_model_files.append((file_path, None, error)) - - readable_contents: list[str] = [ - contents - for _path, contents, error in loaded_model_files - if contents is not None and error is None - ] - readable_matches: Iterator[ModelHeaderMatch | None] = iter( - match_model_headers(readable_contents) ) - header_matches: list[ModelHeaderMatch | None] = [ - next(readable_matches) if contents is not None and error is None else None - for _path, contents, error in loaded_model_files - ] - prepare_matched_model_file_headers(header_matches) - discovered_model_files: list[DiscoveredSqlModelFile] = [] - contents: str | None - read_error: Exception | None - header_match: ModelHeaderMatch | None - for (file_path, contents, read_error), header_match in zip( - loaded_model_files, header_matches, strict=True - ): - if read_error is not None: - if on_fault is None: - raise read_error - on_fault(_discovery_fault(project_dir=project_dir, path=file_path, error=read_error)) - continue - if contents is None: - raise ModelSqlParseError("Model file read returned neither contents nor an error") - try: - discovered_model_files.append( - discover_matched_model_file( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - header_match=header_match, - extract_implicit_alias_columns=extract_implicit_alias_columns, - extract_output_column_locations=extract_output_column_locations, - ) - ) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - if on_fault is None: - raise - on_fault(_discovery_fault(project_dir=project_dir, path=file_path, error=error)) - continue - return tuple(discovered_model_files) + tree.memo[memo_key] = facts + return facts def discover_enum_files( @@ -681,52 +413,6 @@ def discover_python_function_files( return tuple(discovered_function_files) -def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: - """Discover model schema.yml files and seed declaration .yml files.""" - - deduped_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) - discovered_schema_files: list[DiscoveredSchemaFile] = [] - file_path: Path - for file_path in deduped_paths: - contents: str = file_path.read_text(encoding="utf-8") - model_entries: tuple[SchemaModelEntry, ...] - seed_entries: tuple[SchemaSeedEntry, ...] - model_entries, seed_entries = parse_schema_yml(contents=contents, file_path=file_path) - discovered_schema_files.append( - DiscoveredSchemaFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - model_entries=model_entries, - seed_entries=seed_entries, - ) - ) - return tuple(discovered_schema_files) - - -def discover_source_files( - *, - project_dir: Path, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - fact_cache: FactCacheStore | None = None, -) -> tuple[DiscoveredSourceFile, ...]: - """Discover source declaration YAML files under sources/.""" - - yaml_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) - - def parse(file_path: Path) -> DiscoveredSourceFile: - return parse_source_file_with_cache( - project_dir=project_dir, - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - fact_cache=fact_cache, - ) - - return _parse_discovered_paths( - project_dir=project_dir, file_paths=yaml_paths, parse=parse, on_fault=on_fault - ) - - def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: """Discover seed CSV files under seeds/.""" @@ -744,57 +430,6 @@ def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: ) -def discover_test_files( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None = None, - on_fault: Callable[[DiscoveryFileFault], None] | None = None, - fact_cache: FactCacheStore | None = None, -) -> tuple[DiscoveredSqlTestFile, ...]: - """Discover SQL-native unit test files under tests/unit/.""" - - tests_root: Path = project_dir / "tests" / "unit" - if not tests_root.is_dir(): - return () - - file_paths: tuple[Path, ...] = tuple( - file_path - for file_path in _sorted_glob(project_dir=project_dir, root=tests_root, pattern="*.sql") - if (selected_paths is None or file_path.resolve() in selected_paths) - and not is_in_scoped_declaration_tree(file_path=file_path, project_dir=project_dir) - ) - return discover_sql_test_files( - project_dir=project_dir, - file_paths=file_paths, - on_fault=on_fault, - fact_cache=fact_cache, - ) - - -def discover_scenario_files( - *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None -) -> tuple[DiscoveredSqlScenarioFile, ...]: - """Discover SQL-native scenario files under tests/scenarios/.""" - - scenarios_root: Path = project_dir / "tests" / "scenarios" - if not scenarios_root.is_dir(): - return () - - def parse(file_path: Path) -> DiscoveredSqlScenarioFile: - return parse_sql_scenario_file( - contents=file_path.read_text(encoding="utf-8"), - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - ) - - return _parse_discovered_paths( - project_dir=project_dir, - file_paths=_unscoped_files(root=scenarios_root, pattern="*.sql", project_dir=project_dir), - parse=parse, - on_fault=on_fault, - ) - - def discover_audit_files( *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None ) -> tuple[DiscoveredAuditFile, ...]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py deleted file mode 100644 index 7a31694a5c..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/model_files.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Assembly of one discovered SQL model file from its pre-matched MODEL header.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.sql.declarations import ( - parse_model_constant_declarations, - parse_model_enum_declarations, -) -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - matched_model_header_column_locations, - matched_model_output_column_locations, - parse_matched_model_sql, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, ModelHeaderMatch - - -def discover_matched_model_file( - *, - file_path: Path, - relative_path: Path, - contents: str, - header_match: ModelHeaderMatch | None, - extract_implicit_alias_columns: bool, - extract_output_column_locations: bool, -) -> DiscoveredSqlModelFile: - """Build one model file from contents whose MODEL header was matched exactly once.""" - - header_values, query_sql = parse_matched_model_sql( - header_match=header_match, file_path=file_path - ) - return DiscoveredSqlModelFile( - file_path=file_path, - relative_path=relative_path, - contents=contents, - header_values=header_values, - header_column_locations=matched_model_header_column_locations( - contents=contents, header_match=header_match, relative_path=relative_path - ), - output_column_locations=( - matched_model_output_column_locations( - contents=contents, - header_match=header_match, - relative_path=relative_path, - extract_implicit_alias_columns=extract_implicit_alias_columns, - ) - if extract_output_column_locations - else {} - ), - query_sql=query_sql, - enum_declarations=parse_model_enum_declarations( - raw_value=header_values.get("enums"), - model_name=file_path.stem, - relative_path=relative_path, - ), - constant_declarations=parse_model_constant_declarations( - raw_value=header_values.get("constants"), - model_name=file_path.stem, - relative_path=relative_path, - ), - extract_implicit_alias_columns=extract_implicit_alias_columns, - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py index 0439371b7c..d833e2b626 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/named_declarations.py @@ -2,35 +2,19 @@ from __future__ import annotations -from collections.abc import Iterable, Iterator +from collections.abc import Iterator from pathlib import Path -from typing import cast +from sqlbuild.compiler.discovery._helpers.native.declarations import native_declaration_groups from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.constants import CANONICAL_AUTHORED_ROOTS -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import NamedDeclarationRoot from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - GLOBAL_NAMED_DECLARATION_DIRECTORIES, GLOBAL_NAMED_DECLARATION_ROOTS, GROUPED_NAMED_DECLARATION_ROLES, - INHERITED_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind -_AUDIT_ROLE_DIRECTORY: str = "audits" -_LOCAL_AUDIT_ROLE_DIRECTORY: str = "_audits" -_SINGULAR_AUDIT_DIRECTORY: str = "singular" -_LAYOUT_MEMO_KEY: str = "named_declaration_layout" -_GROUPS_MEMO_KEY: str = "declaration_groups" _PATH_SEPARATOR: str = "/" -_NESTED_ROLE_DIRECTORIES: frozenset[str] = frozenset( - {DECLARATION_GROUP_DIRECTORY} - | INHERITED_DECLARATION_DIRECTORIES - | LOCAL_DECLARATION_DIRECTORIES -) def named_declaration_roots( @@ -38,10 +22,6 @@ def named_declaration_roots( ) -> tuple[NamedDeclarationRoot, ...]: """Return validated top-level and grouped declaration roots for the requested kinds.""" - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - if _LAYOUT_MEMO_KEY not in tree.memo: - validate_named_declaration_layout(project_dir=project_dir) - tree.memo[_LAYOUT_MEMO_KEY] = True roots: list[NamedDeclarationRoot] = [] for role_parts, kind in GLOBAL_NAMED_DECLARATION_ROOTS.items(): directory: Path = project_dir.joinpath(*role_parts) @@ -87,111 +67,8 @@ def named_declaration_files( yield root, file_path -def remember_declaration_groups(*, project_dir: Path, groups: Iterable[tuple[str, str]]) -> None: - """Record a named layout another walk of this pass validated, with its declaration groups.""" - - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - tree.memo[_GROUPS_MEMO_KEY] = tuple( - (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) for root, directory in groups - ) - tree.memo[_LAYOUT_MEMO_KEY] = True - - -def validate_named_declaration_layout(*, project_dir: Path) -> None: - """Reject entries that no named declaration role accepts.""" - - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - audits_root: Path = project_dir / _AUDIT_ROLE_DIRECTORY - if audits_root.is_dir(): - _require_role_children( - directory=audits_root, - allowed=frozenset({"generic", _SINGULAR_AUDIT_DIRECTORY}), - project_dir=project_dir, - help_text="audit files must live in audits/generic/ or audits/singular/", - ) - for top_level in sorted(GLOBAL_NAMED_DECLARATION_DIRECTORIES): - directory: Path = project_dir / top_level - if not directory.is_dir(): - continue - for nested in sorted(tree.directories(root=directory)): - if nested.name in _NESTED_ROLE_DIRECTORIES: - nested_path: str = _relative(path=nested, project_dir=project_dir) - raise DeclarationParseError( - f"Declaration directory {nested_path}/ is not allowed " - f"inside the project-wide {top_level}/ role; scoped declarations belong " - f"under /{DECLARATION_GROUP_DIRECTORY}/ below a resource tree" - ) - for _root_components, group in _declaration_groups(project_dir=project_dir): - for role_directory, allowed in ( - (group / _AUDIT_ROLE_DIRECTORY, frozenset({"generic", _SINGULAR_AUDIT_DIRECTORY})), - (group / _LOCAL_AUDIT_ROLE_DIRECTORY, frozenset({"generic"})), - (group / "hooks", frozenset({"sql", "python"})), - (group / "_hooks", frozenset({"sql", "python"})), - ): - if not role_directory.is_dir(): - continue - if ( - role_directory.name == _LOCAL_AUDIT_ROLE_DIRECTORY - and (role_directory / _SINGULAR_AUDIT_DIRECTORY).exists() - ): - singular_path: str = _relative( - path=role_directory / _SINGULAR_AUDIT_DIRECTORY, project_dir=project_dir - ) - raise DeclarationParseError( - f"{singular_path}/ is " - "invalid: singular audits are never used by name, so folder-only visibility " - f"has no meaning; use {DECLARATION_GROUP_DIRECTORY}/audits/singular/" - ) - _require_role_children( - directory=role_directory, - allowed=allowed, - project_dir=project_dir, - help_text=( - f"{role_directory.name}/ accepts only " - + ", ".join(f"{name}/" for name in sorted(allowed)) - ), - ) - - -def _require_role_children( - *, directory: Path, allowed: frozenset[str], project_dir: Path, help_text: str -) -> None: - unsupported: tuple[Path, ...] = tuple( - sorted( - child - for child in directory.iterdir() - if not child.name.startswith(".") and (not child.is_dir() or child.name not in allowed) - ) - ) - if unsupported: - rendered: str = ", ".join( - _relative(path=path, project_dir=project_dir) for path in unsupported - ) - directory_path: str = _relative(path=directory, project_dir=project_dir) - raise DeclarationParseError( - f"Unsupported entries in {directory_path}/: {rendered}; {help_text}" - ) - - def _declaration_groups(*, project_dir: Path) -> tuple[tuple[tuple[str, ...], Path], ...]: - tree: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) - cached: object = tree.memo.get(_GROUPS_MEMO_KEY) - if cached is not None: - return cast(tuple[tuple[tuple[str, ...], Path], ...], cached) - groups: list[tuple[tuple[str, ...], Path]] = [] - for root_components in CANONICAL_AUTHORED_ROOTS: - authored_root: Path = project_dir.joinpath(*root_components) - if not authored_root.is_dir(): - continue - groups.extend( - (root_components, group) - for group in sorted(tree.rglob(root=authored_root, pattern=DECLARATION_GROUP_DIRECTORY)) - if group.is_dir() and group.parent != authored_root - ) - result: tuple[tuple[tuple[str, ...], Path], ...] = tuple(groups) - tree.memo[_GROUPS_MEMO_KEY] = result - return result - - -def _relative(*, path: Path, project_dir: Path) -> str: - return path.relative_to(project_dir).as_posix() + return tuple( + (tuple(root.split(_PATH_SEPARATOR)), project_dir / directory) + for root, directory in native_declaration_groups(project_dir=project_dir) + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/sql_test_files.py deleted file mode 100644 index c887759507..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/sql_test_files.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Filesystem discovery for SQL-native unit test files.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( - decode_cached_sql_test_file, - encode_cached_sql_test_file, -) -from sqlbuild.compiler.discovery._helpers.sql.tests import ( - parse_sql_test_file, - prepare_sql_test_file_headers, -) -from sqlbuild.compiler.discovery.constants import ( - DISCOVERY_SQL_TEST_FACT_KIND, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.exceptions import SqlTestParseError -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, - DiscoveryFileFault, -) -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore - -_TEST_FILE_BATCH_SIZE: int = 512 - - -def discover_sql_test_files( - *, - project_dir: Path, - file_paths: tuple[Path, ...], - on_fault: Callable[[DiscoveryFileFault], None] | None, - fact_cache: FactCacheStore | None = None, -) -> tuple[DiscoveredSqlTestFile, ...]: - """Read and parse ordered SQL test paths through bounded native header batches.""" - - loaded: list[tuple[Path, str | None, Exception | None]] = [] - for file_path in file_paths: - try: - loaded.append((file_path, file_path.read_text(encoding="utf-8"), None)) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - loaded.append((file_path, None, error)) - - cache_keys: dict[Path, str] = ( - { - file_path: fact_cache.key( - DISCOVERY_SQL_TEST_FACT_KIND, str(project_dir), str(file_path), contents - ) - for file_path, contents, error in loaded - if contents is not None and error is None - } - if fact_cache is not None and fact_cache.enabled - else {} - ) - cached_facts: dict[str, object] = ( - fact_cache.read_many( - tuple((_fact_slot(file_path), cache_key) for file_path, cache_key in cache_keys.items()) - ) - if fact_cache is not None - else {} - ) - discovered: list[DiscoveredSqlTestFile] = [] - for batch_start in range(0, len(loaded), _TEST_FILE_BATCH_SIZE): - batch: list[tuple[Path, str | None, Exception | None]] = loaded[ - batch_start : batch_start + _TEST_FILE_BATCH_SIZE - ] - _prepare_batch([item for item in batch if cache_keys.get(item[0]) not in cached_facts]) - for file_path, contents, read_error in batch: - if read_error is not None: - if on_fault is None: - raise read_error - on_fault(_file_fault(project_dir=project_dir, path=file_path, error=read_error)) - continue - if contents is None: - raise SqlTestParseError("SQL test file read returned neither contents nor an error") - cache_key: str | None = cache_keys.get(file_path) - cached_test_file: DiscoveredSqlTestFile | None = ( - decode_cached_sql_test_file( - fact=cached_facts.get(cache_key), file_path=file_path, contents=contents - ) - if cache_key is not None - else None - ) - if cached_test_file is not None: - discovered.append(cached_test_file) - continue - try: - blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file( - contents=contents, file_path=file_path - ) - except (OSError, UnicodeError, ValueError, SyntaxError) as error: - if on_fault is None: - raise - on_fault(_file_fault(project_dir=project_dir, path=file_path, error=error)) - continue - test_file: DiscoveredSqlTestFile = DiscoveredSqlTestFile( - file_path=file_path, - relative_path=file_path.relative_to(project_dir), - contents=contents, - blocks=blocks, - ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), - ) - discovered.append(test_file) - if fact_cache is not None and cache_key is not None: - fact_cache.stage( - key=cache_key, - slot=_fact_slot(file_path), - value=encode_cached_sql_test_file(test_file), - ) - return tuple(discovered) - - -def _fact_slot(file_path: Path) -> str: - return f"{DISCOVERY_SQL_TEST_FACT_KIND}:{file_path}" - - -def _prepare_batch(batch: list[tuple[Path, str | None, Exception | None]]) -> None: - contents_batch: list[str] = [] - for _path, contents, error in batch: - if contents is not None and error is None: - contents_batch.append(contents) - try: - prepare_sql_test_file_headers(contents_batch) - except (OSError, UnicodeError, ValueError, SyntaxError): - pass - - -def _file_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault: - try: - relative_path: Path | None = path.relative_to(project_dir) - except ValueError: - relative_path = None - return DiscoveryFileFault( - path=relative_path, - message=str(error).replace(str(project_dir), "."), - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py index 2bb75590b7..29b613e0ed 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declarations.py @@ -1,39 +1,67 @@ -"""The declaration layout walked and validated natively for the Python declaration collections.""" +"""The declaration layout walked and validated natively for the declaration collections.""" from __future__ import annotations from pathlib import Path +from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import remember_declaration_file_facts -from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( - remember_declaration_groups, -) from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_discovery_supported, - native_display_prefix, + native_collection, + native_failure, native_project_tree, seed_snapshot_listings, ) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.constants import NATIVE_FAILED_TAG from sqlbuild.compiler.discovery.types import NativeDeclarationFact +from sqlbuild.compiler.scopes.types import DeclarationKind + +_LAYOUT_MEMO_KEY: str = "native_declaration_layout" +_GROUPS_MEMO_KEY: str = "native_declaration_groups" + + +def native_declaration_file_facts( + *, project_dir: Path, declaration_kind: DeclarationKind | None +) -> list[NativeDeclarationFact]: + """Return the declaration files of one kind (or every kind), raising an invalid layout.""" + + facts, _groups = _layout(project_dir=project_dir, declaration_kind=declaration_kind) + return cast(list[NativeDeclarationFact], _rows(facts)) -def prepare_native_declaration_layout(*, project_dir: Path) -> None: - """Record a natively validated declaration layout; Python rescans an invalid one to raise.""" +def native_declaration_groups(*, project_dir: Path) -> list[tuple[str, str]]: + """Return the `(canonical root, group directory)` pairs of a valid named layout.""" - if not native_discovery_supported( - project_dir=project_dir, display_prefix=native_display_prefix(project_dir) - ): - return + snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + groups: object = snapshot.memo.get(_GROUPS_MEMO_KEY) + if groups is None: + _facts, groups = _layout(project_dir=project_dir, declaration_kind=None) + return cast(list[tuple[str, str]], _rows(cast(tuple[object, ...], groups))) + + +def _layout( + *, project_dir: Path, declaration_kind: DeclarationKind | None +) -> tuple[tuple[object, ...], tuple[object, ...]]: + snapshot: DirectorySnapshot = DirectorySnapshot.current(project_dir=project_dir) + memo_key: tuple[str, DeclarationKind | None] = (_LAYOUT_MEMO_KEY, declaration_kind) + cached: object = snapshot.memo.get(memo_key) + if cached is not None: + return cast(tuple[tuple[object, ...], tuple[object, ...]], cached) tree: _native.NativeProjectTree = native_project_tree(project_dir) - layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( - _native.discover_declaration_layout(tree) + layout: tuple[tuple[object, ...], tuple[object, ...]] = native_collection( + result=_native.discover_declaration_layout( + tree, None if declaration_kind is None else declaration_kind.value + ), + project_dir=project_dir, ) seed_snapshot_listings(project_dir=project_dir, tree=tree) - if layout is None: - return - facts, groups = layout - if facts is not None: - remember_declaration_file_facts(project_dir=project_dir, facts=facts) - if groups is not None: - remember_declaration_groups(project_dir=project_dir, groups=groups) + snapshot.memo[memo_key] = layout + _ = snapshot.memo.setdefault(_GROUPS_MEMO_KEY, layout[1]) + return layout + + +def _rows(payload: tuple[object, ...]) -> object: + if payload[0] == NATIVE_FAILED_TAG: + raise native_failure(payload) + return payload[1] diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index 8f0ea0ee33..4c166f2493 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -2,39 +2,32 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files -from sqlbuild.compiler.discovery._helpers.filesystem.model_files import ( - discover_matched_model_file, -) from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_discovery_supported, - native_display_prefix, - native_failure, + materialise_native_files, + native_collection, native_locations, + native_payload_error, native_project_tree, - native_text_runtime, + native_request, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.declarations import ( parse_model_constant_declarations, parse_model_enum_declarations, ) -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - match_model_header, - project_native_header_values, -) +from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_UNREADABLE_TAG, REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.discovery.types import NativeLocation +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveryFileFault +from sqlbuild.compiler.discovery.types import NativeFiles, NativeLocation def discover_native_model_files( @@ -42,60 +35,44 @@ def discover_native_model_files( project_dir: Path, extract_implicit_alias_columns: bool, extract_output_column_locations: bool, + selected_model_names: frozenset[str] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, ) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover model files natively; Python runs when native discovery cannot match it.""" + """Discover the (selected) model files natively, reporting failing files to `on_fault`.""" - display_prefix: str = native_display_prefix(project_dir) - files: list[tuple[str, tuple[object, ...]]] | None = ( - _discover_native_files( + with DirectorySnapshot.scope(project_dir=project_dir): + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: NativeFiles = native_collection( + result=_native.discover_model_files( + native_request( + project_dir=project_dir, + fields={ + "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), + "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), + "extract_implicit_alias_columns": extract_implicit_alias_columns, + "extract_output_column_locations": extract_output_column_locations, + }, + ), + tree, + ), project_dir=project_dir, - display_prefix=display_prefix, - extract_implicit_alias_columns=extract_implicit_alias_columns, - extract_output_column_locations=extract_output_column_locations, ) - if native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix) - else None - ) - if files is None: - return discover_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=extract_implicit_alias_columns, - extract_output_column_locations=extract_output_column_locations, - ) - return tuple( - _model_file( + seed_snapshot_listings(project_dir=project_dir, tree=tree) + return materialise_native_files( + project_dir=project_dir, + files=( + (Path(relative_path), payload) + for relative_path, payload in files + if selected_model_names is None or Path(relative_path).stem in selected_model_names + ), + build=lambda relative_path, payload: _model_file( project_dir=project_dir, - relative_path=Path(relative_path), + relative_path=relative_path, payload=payload, extract_implicit_alias_columns=extract_implicit_alias_columns, - extract_output_column_locations=extract_output_column_locations, - ) - for relative_path, payload in files - ) - - -def _discover_native_files( - *, - project_dir: Path, - display_prefix: str, - extract_implicit_alias_columns: bool, - extract_output_column_locations: bool, -) -> list[tuple[str, tuple[object, ...]]] | None: - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: list[tuple[str, tuple[object, ...]]] | None = _native.discover_model_files( - { - "project_dir": str(project_dir), - "display_prefix": display_prefix, - "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), - "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), - "extract_implicit_alias_columns": extract_implicit_alias_columns, - "extract_output_column_locations": extract_output_column_locations, - **native_text_runtime(), - }, - tree, + ), + on_fault=on_fault, ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) - return files def _model_file( @@ -104,22 +81,11 @@ def _model_file( relative_path: Path, payload: tuple[object, ...], extract_implicit_alias_columns: bool, - extract_output_column_locations: bool, ) -> DiscoveredSqlModelFile: file_path: Path = project_dir / relative_path - tag: object = payload[0] - if tag == NATIVE_UNREADABLE_TAG: - contents: str = file_path.read_text(encoding="utf-8") - return discover_matched_model_file( - file_path=file_path, - relative_path=relative_path, - contents=contents, - header_match=match_model_header(contents), - extract_implicit_alias_columns=extract_implicit_alias_columns, - extract_output_column_locations=extract_output_column_locations, - ) - if tag == NATIVE_FAILED_TAG: - raise native_failure(payload) + error: Exception | None = native_payload_error(payload=payload, file_path=file_path) + if error is not None: + raise error _tag, contents, values, header_locations, output_locations, query_sql = payload header_values: dict[str, object] = project_native_header_values(cast(dict[str, object], values)) return DiscoveredSqlModelFile( diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index e72357241a..27bf2066e9 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -5,17 +5,30 @@ import os import sys import unicodedata +from collections.abc import Callable, Iterable from pathlib import Path +from typing import cast import sqlbuild._native as _native from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot +from sqlbuild.compiler.discovery.constants import ( + NATIVE_FAILED_TAG, + NATIVE_SUPPORTED_PYTHON_VERSIONS, + NATIVE_UNLISTABLE_TAG, + NATIVE_UNREADABLE_TAG, +) from sqlbuild.compiler.discovery.exceptions import ( DeclarationParseError, DiscoveryError, ModelSqlParseError, + ProjectPathError, + SchemaParseError, + SourceParseError, SqlScenarioParseError, SqlTestParseError, + UnsupportedPythonError, ) +from sqlbuild.compiler.discovery.models import DiscoveryFileFault from sqlbuild.compiler.discovery.types import ( DirectorySnapshotEntry, NativeListing, @@ -24,14 +37,25 @@ from sqlbuild.spec.contracts.models import SourceLocation _DISPLAY_PROBE: str = "_" -_WINDOWS_OS_NAME: str = "nt" _PATH_ENCODING: str = "utf-8" +_DECODE_ENCODING: str = "utf-8" +_DECODE_READ_KIND: str = "decode" _NATIVE_TREE_MEMO_KEY: str = "native_project_tree" +_PROJECT_PATH_KIND: str = "project_path" +_NAMES_ARE_BYTES: bool = os.supports_bytes_environ +_SURROGATE_ESCAPE_BASE: int = 0xDC00 +_ESCAPED_BYTE_FIRST: int = 0xDC80 +_ESCAPED_BYTE_LAST: int = 0xDCFF +_SURROGATE_FIRST: int = 0xD800 +_SURROGATE_LAST: int = 0xDFFF _FAILURE_CLASSES: dict[str, type[DiscoveryError]] = { "model_sql": ModelSqlParseError, "declaration": DeclarationParseError, "sql_test": SqlTestParseError, "sql_scenario": SqlScenarioParseError, + "schema": SchemaParseError, + "source": SourceParseError, + _PROJECT_PATH_KIND: ProjectPathError, } @@ -49,6 +73,74 @@ def native_failure(payload: tuple[object, ...]) -> DiscoveryError: return error_class(str(message), help=None if help_text is None else str(help_text)) +def native_read_error(*, payload: tuple[object, ...], file_path: Path) -> Exception: + """Build the error Python's `read_text` raises for a file native discovery could not read.""" + + if payload[1] == _DECODE_READ_KIND: + _tag, _kind, data, start, end, reason = payload + return UnicodeDecodeError( + _DECODE_ENCODING, cast(bytes, data), cast(int, start), cast(int, end), str(reason) + ) + _tag, _kind, errno, message, winerror = payload + if winerror is not None: + return OSError(None, str(message), str(file_path), cast(int, winerror)) + if errno is None: + return OSError(str(message)) + return OSError(errno, os.strerror(cast(int, errno)), str(file_path)) + + +def native_payload_error(*, payload: tuple[object, ...], file_path: Path) -> Exception | None: + """Return the error a failed or unreadable file payload raises, or `None` for a parsed one.""" + + tag: object = payload[0] + if tag == NATIVE_FAILED_TAG: + return native_failure(payload) + if tag == NATIVE_UNREADABLE_TAG: + return native_read_error(payload=payload, file_path=file_path) + return None + + +def native_collection[ResultT]( + *, result: ResultT | tuple[object, ...], project_dir: Path +) -> ResultT: + """Return a native collection, raising the error of a collection that could not be read.""" + + if not isinstance(result, tuple) or not isinstance(result[0], str): + return cast(ResultT, result) + payload: tuple[object, ...] = result + if payload[0] == NATIVE_UNLISTABLE_TAG: + _tag, relative_path, read_payload = payload + raise native_read_error( + payload=cast(tuple[object, ...], read_payload), + file_path=project_dir / str(relative_path), + ) + raise native_failure(payload) + + +def materialise_native_files[RecordT]( + *, + project_dir: Path, + files: Iterable[tuple[Path, tuple[object, ...]]], + build: Callable[[Path, tuple[object, ...]], RecordT], + on_fault: Callable[[DiscoveryFileFault], None] | None, +) -> tuple[RecordT, ...]: + """Build each native file payload, reporting and skipping a failing file when asked.""" + + records: list[RecordT] = [] + for relative_path, payload in files: + try: + records.append(build(relative_path, payload)) + except (OSError, UnicodeError, ValueError, SyntaxError) as error: + if on_fault is None: + raise + on_fault( + DiscoveryFileFault( + path=relative_path, message=str(error).replace(str(project_dir), ".") + ) + ) + return tuple(records) + + def native_locations( *, locations: list[NativeLocation], relative_path: Path ) -> dict[str, SourceLocation]: @@ -69,18 +161,14 @@ def native_locations( def native_text_runtime() -> dict[str, object]: """Return the request fields naming the Python whose string semantics native reproduces.""" - return {"python_version": _python_version(), "unicode_version": unicodedata.unidata_version} - - -def native_discovery_supported(*, project_dir: Path, display_prefix: str) -> bool: - """Whether native discovery reproduces Python here: known Python, UTF-8 paths, POSIX.""" - - return ( - os.name != _WINDOWS_OS_NAME - and _native.native_text_supported(_python_version(), unicodedata.unidata_version) - and _is_utf8_text(str(project_dir)) - and _is_utf8_text(display_prefix) - ) + python_version: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) + if not _native.native_text_supported(python_version, unicodedata.unidata_version): + raise UnsupportedPythonError( + f"This SQLBuild release supports Python {NATIVE_SUPPORTED_PYTHON_VERSIONS}, not " + f"Python {python_version[0]}.{python_version[1]} (Unicode " + f"{unicodedata.unidata_version}); run sqb with a supported Python" + ) + return {"python_version": python_version, "unicode_version": unicodedata.unidata_version} def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: @@ -90,11 +178,28 @@ def native_project_tree(project_dir: Path) -> _native.NativeProjectTree: tree: object = snapshot.memo.get(_NATIVE_TREE_MEMO_KEY) if isinstance(tree, _native.NativeProjectTree): return tree + for text in (str(project_dir), native_display_prefix(project_dir)): + if not native_path_text_supported(text): + raise ProjectPathError( + f"Project directory {_escaped(text)} is not valid UTF-8; move the project to a " + "directory whose path is valid UTF-8" + ) created: _native.NativeProjectTree = _native.NativeProjectTree(str(project_dir)) snapshot.memo[_NATIVE_TREE_MEMO_KEY] = created return created +def native_request(*, project_dir: Path, fields: dict[str, object]) -> dict[str, object]: + """Return a native discovery request for the project with the running Python's semantics.""" + + return { + "project_dir": str(project_dir), + "display_prefix": native_display_prefix(project_dir), + **fields, + **native_text_runtime(), + } + + def seed_snapshot_listings(*, project_dir: Path, tree: _native.NativeProjectTree) -> None: """Share the native walk's listings with the pass's Python directory snapshot.""" @@ -116,10 +221,6 @@ def _snapshot_entries( def native_path_text_supported(text: str) -> bool: """Whether a path's text crosses to the native engine unchanged (no surrogate escapes).""" - return _is_utf8_text(text) - - -def _is_utf8_text(text: str) -> bool: try: _ = text.encode(_PATH_ENCODING) except UnicodeEncodeError: @@ -127,5 +228,28 @@ def _is_utf8_text(text: str) -> bool: return True -def _python_version() -> tuple[int, int]: - return (sys.version_info[0], sys.version_info[1]) +def native_unreadable_path_payload(*, project_dir: Path, relative_text: str) -> tuple[object, ...]: + """Return the per-file failure payload for a path native discovery cannot name, if any.""" + + if native_path_text_supported(relative_text): + return () + return ( + NATIVE_FAILED_TAG, + _PROJECT_PATH_KIND, + f"Project path {_escaped(str(project_dir / relative_text))} is not valid UTF-8; rename it " + "so SQLBuild can read it", + None, + ) + + +def _escaped(text: str) -> str: + return "".join(_escaped_character(character) for character in text) + + +def _escaped_character(character: str) -> str: + code_point: int = ord(character) + if _NAMES_ARE_BYTES and _ESCAPED_BYTE_FIRST <= code_point <= _ESCAPED_BYTE_LAST: + return f"\\x{code_point - _SURROGATE_ESCAPE_BASE:02x}" + if _SURROGATE_FIRST <= code_point <= _SURROGATE_LAST: + return f"\\u{code_point:04x}" + return character diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py index 89391f59a2..b161912ba6 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py @@ -7,31 +7,24 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_scenario_files, - discover_test_files, -) from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_discovery_supported, - native_display_prefix, + materialise_native_files, + native_collection, native_failure, + native_payload_error, native_project_tree, + native_request, native_text_runtime, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values -from sqlbuild.compiler.discovery._helpers.sql.scenarios import ( - build_sql_scenario_file, - parse_sql_scenario_file, -) +from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file from sqlbuild.compiler.discovery._helpers.sql.tests import ( build_sql_test_block, - parse_sql_test_file, validate_sql_test_names, ) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( - NATIVE_FAILED_TAG, - NATIVE_UNREADABLE_TAG, SQL_SCENARIO_HEADER_KEYS, SQL_TEST_HEADER_KEYS, SQL_TESTS_OWNERSHIP_ROOT, @@ -40,61 +33,104 @@ DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, + DiscoveryFileFault, ) -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.discovery.types import NativeFiles -type _NativeFiles = list[tuple[str, tuple[object, ...]]] -type _NativeDiscovery = Callable[ - [dict[str, object], _native.NativeProjectTree], _NativeFiles | None -] +type _NativeDiscovery = Callable[[dict[str, object], _native.NativeProjectTree], object] def discover_native_test_files( - *, project_dir: Path, fact_cache: FactCacheStore | None = None + *, + project_dir: Path, + selected_paths: frozenset[Path] | None = None, + on_fault: Callable[[DiscoveryFileFault], None] | None = None, ) -> tuple[DiscoveredSqlTestFile, ...]: - """Discover SQL test files natively; Python, with its fact cache, runs where native cannot.""" + """Discover the (selected) SQL test files natively, reporting failing files to `on_fault`.""" - files: _NativeFiles | None = _native_files( + files: NativeFiles = _native_files( project_dir=project_dir, discover=_native.discover_sql_test_files ) - if files is None: - return discover_test_files(project_dir=project_dir, fact_cache=fact_cache) - return tuple( - _test_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) - for relative_path, payload in files + return materialise_native_files( + project_dir=project_dir, + files=( + (Path(relative_path), payload) + for relative_path, payload in files + if selected_paths is None or (project_dir / relative_path).resolve() in selected_paths + ), + build=lambda relative_path, payload: _test_file( + project_dir=project_dir, relative_path=relative_path, payload=payload + ), + on_fault=on_fault, ) -def discover_native_scenario_files(*, project_dir: Path) -> tuple[DiscoveredSqlScenarioFile, ...]: - """Discover scenario files natively; Python runs when native discovery cannot match it.""" +def discover_native_scenario_files( + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None +) -> tuple[DiscoveredSqlScenarioFile, ...]: + """Discover scenario files natively, reporting failing files to `on_fault`.""" - files: _NativeFiles | None = _native_files( + files: NativeFiles = _native_files( project_dir=project_dir, discover=_native.discover_scenario_files ) - if files is None: - return discover_scenario_files(project_dir=project_dir) - return tuple( - _scenario_file(project_dir=project_dir, relative_path=Path(relative_path), payload=payload) - for relative_path, payload in files + return materialise_native_files( + project_dir=project_dir, + files=((Path(relative_path), payload) for relative_path, payload in files), + build=lambda relative_path, payload: _scenario_file( + project_dir=project_dir, relative_path=relative_path, payload=payload + ), + on_fault=on_fault, ) -def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> _NativeFiles | None: - display_prefix: str = native_display_prefix(project_dir) - if not native_discovery_supported(project_dir=project_dir, display_prefix=display_prefix): - return None - tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: _NativeFiles | None = discover( - { - "project_dir": str(project_dir), - "display_prefix": display_prefix, - "test_keys": sorted(SQL_TEST_HEADER_KEYS), - "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), - **native_text_runtime(), - }, - tree, +def parse_native_sql_test_contents( + *, contents: str, file_path: Path +) -> tuple[DiscoveredSqlTestBlock, ...]: + """Parse in-memory SQL test contents into ordered blocks, as test discovery does.""" + + payload: tuple[object, ...] = _native.parse_sql_test_contents( + _text_request(file_path), contents + ) + error: Exception | None = native_payload_error(payload=payload, file_path=file_path) + if error is not None: + raise error + _tag, _contents, native_blocks, block_failure = payload + return _test_blocks( + file_path=file_path, + native_blocks=cast(list[tuple[dict[str, object], str]], native_blocks), + block_failure=cast(tuple[object, ...] | None, block_failure), ) - seed_snapshot_listings(project_dir=project_dir, tree=tree) + + +def _text_request(file_path: Path) -> dict[str, object]: + return { + "file_path": str(file_path), + "test_keys": sorted(SQL_TEST_HEADER_KEYS), + "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + **native_text_runtime(), + } + + +def _native_files(*, project_dir: Path, discover: _NativeDiscovery) -> NativeFiles: + with DirectorySnapshot.scope(project_dir=project_dir): + tree: _native.NativeProjectTree = native_project_tree(project_dir) + files: NativeFiles = native_collection( + result=cast( + NativeFiles | tuple[object, ...], + discover( + native_request( + project_dir=project_dir, + fields={ + "test_keys": sorted(SQL_TEST_HEADER_KEYS), + "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + }, + ), + tree, + ), + ), + project_dir=project_dir, + ) + seed_snapshot_listings(project_dir=project_dir, tree=tree) return files @@ -102,27 +138,19 @@ def _test_file( *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] ) -> DiscoveredSqlTestFile: file_path: Path = project_dir / relative_path - tag: object = payload[0] - if tag == NATIVE_FAILED_TAG: - raise native_failure(payload) - contents: str - blocks: tuple[DiscoveredSqlTestBlock, ...] - if tag == NATIVE_UNREADABLE_TAG: - contents = file_path.read_text(encoding="utf-8") - blocks = parse_sql_test_file(contents=contents, file_path=file_path) - else: - _tag, native_contents, native_blocks, block_failure = payload - contents = str(native_contents) - blocks = _test_blocks( - file_path=file_path, - native_blocks=cast(list[tuple[dict[str, object], str]], native_blocks), - block_failure=cast(tuple[object, ...] | None, block_failure), - ) + error: Exception | None = native_payload_error(payload=payload, file_path=file_path) + if error is not None: + raise error + _tag, native_contents, native_blocks, block_failure = payload return DiscoveredSqlTestFile( file_path=file_path, relative_path=relative_path, - contents=contents, - blocks=blocks, + contents=str(native_contents), + blocks=_test_blocks( + file_path=file_path, + native_blocks=cast(list[tuple[dict[str, object], str]], native_blocks), + block_failure=cast(tuple[object, ...] | None, block_failure), + ), ownership_root=Path(SQL_TESTS_OWNERSHIP_ROOT), ) @@ -151,16 +179,17 @@ def _test_blocks( def _scenario_file( *, project_dir: Path, relative_path: Path, payload: tuple[object, ...] ) -> DiscoveredSqlScenarioFile: - file_path: Path = project_dir / relative_path - tag: object = payload[0] - if tag == NATIVE_FAILED_TAG: - raise native_failure(payload) - if tag == NATIVE_UNREADABLE_TAG: - return parse_sql_scenario_file( - contents=file_path.read_text(encoding="utf-8"), - file_path=file_path, - relative_path=relative_path, - ) + return _scenario_file_from_payload( + payload=payload, file_path=project_dir / relative_path, relative_path=relative_path + ) + + +def _scenario_file_from_payload( + *, payload: tuple[object, ...], file_path: Path, relative_path: Path +) -> DiscoveredSqlScenarioFile: + error: Exception | None = native_payload_error(payload=payload, file_path=file_path) + if error is not None: + raise error _tag, contents, values, sql_body = payload return build_sql_scenario_file( header_values=project_native_header_values(cast(dict[str, object], values)), diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py index a374d1b213..0760756762 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/yaml_files.py @@ -1,154 +1,151 @@ -"""Source and schema YAML files loaded by the native engine, with per-file Python loading.""" +"""Source and schema YAML files read and loaded by the native engine.""" from __future__ import annotations from collections.abc import Callable, Iterator from pathlib import Path -from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( - parse_source_file_with_cache, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_source_files from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import project_relative_path from sqlbuild.compiler.discovery._helpers.native.payloads import ( - native_discovery_supported, + materialise_native_files, + native_collection, native_display_prefix, - native_path_text_supported, + native_payload_error, native_project_tree, + native_unreadable_path_payload, seed_snapshot_listings, ) from sqlbuild.compiler.discovery._helpers.yml.file_paths import ( schema_file_paths, source_file_paths, ) -from sqlbuild.compiler.discovery._helpers.yml.schema import ( - parse_loaded_schema_yml, - parse_schema_yml, -) +from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( - NATIVE_LOADED_TAG, - NATIVE_UNREADABLE_TAG, + NATIVE_SCHEMA_YAML_KIND, + NATIVE_SOURCE_YAML_KIND, NATIVE_YAML_BATCH_BYTES, ) -from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSourceFile -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore -from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry +from sqlbuild.compiler.discovery.models import ( + DiscoveredSchemaFile, + DiscoveredSourceFile, + DiscoveryFileFault, +) type _Payload = tuple[object, ...] def discover_native_source_files( - *, project_dir: Path, fact_cache: FactCacheStore | None = None + *, project_dir: Path, on_fault: Callable[[DiscoveryFileFault], None] | None = None ) -> tuple[DiscoveredSourceFile, ...]: - """Discover source YAML files natively; Python, with its fact cache, loads the rest.""" - - file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) - if not _native_supported(project_dir=project_dir, file_paths=file_paths): - return discover_source_files(project_dir=project_dir, fact_cache=fact_cache) - discovered: list[DiscoveredSourceFile] = [] - for file_path, payload in _native_payloads(project_dir=project_dir, file_paths=file_paths): - relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) - if payload[0] == NATIVE_LOADED_TAG: - _tag, contents, loaded = payload - discovered.append( - DiscoveredSourceFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), - ) - ) - else: - discovered.append( - parse_source_file_with_cache( - project_dir=project_dir, - file_path=file_path, - relative_path=relative_path, - fact_cache=fact_cache, + """Discover source YAML files natively, reporting failing files to `on_fault`.""" + + with DirectorySnapshot.scope(project_dir=project_dir): + file_paths: tuple[Path, ...] = source_file_paths(project_dir=project_dir) + return materialise_native_files( + project_dir=project_dir, + files=( + (project_relative_path(path=file_path, project_dir=project_dir), payload) + for file_path, payload in _native_payloads( + project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SOURCE_YAML_KIND ) - ) - return tuple(discovered) + ), + build=lambda relative_path, payload: _source_file( + project_dir=project_dir, relative_path=relative_path, payload=payload + ), + on_fault=on_fault, + ) def discover_native_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, ...]: - """Discover schema YAML files natively; Python loads any file native cannot reproduce.""" - - file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) - payloads: Iterator[tuple[Path, _Payload]] = ( - _native_payloads(project_dir=project_dir, file_paths=file_paths) - if _native_supported(project_dir=project_dir, file_paths=file_paths) - else ((file_path, (NATIVE_UNREADABLE_TAG,)) for file_path in file_paths) - ) - discovered: list[DiscoveredSchemaFile] = [] - for file_path, payload in payloads: - contents: str - entries: tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]] - if payload[0] == NATIVE_LOADED_TAG: - _tag, native_contents, loaded = payload - contents = str(native_contents) - entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) - else: - contents = _python_contents(file_path=file_path, payload=payload) - entries = parse_schema_yml(contents=contents, file_path=file_path) - discovered.append( - DiscoveredSchemaFile( - file_path=file_path, - relative_path=project_relative_path(path=file_path, project_dir=project_dir), - contents=contents, - model_entries=entries[0], - seed_entries=entries[1], + """Discover model schema.yml and seed declaration files natively.""" + + with DirectorySnapshot.scope(project_dir=project_dir): + file_paths: tuple[Path, ...] = schema_file_paths(project_dir=project_dir) + return tuple( + _schema_file(project_dir=project_dir, file_path=file_path, payload=payload) + for file_path, payload in _native_payloads( + project_dir=project_dir, file_paths=file_paths, kind=NATIVE_SCHEMA_YAML_KIND ) ) - return tuple(discovered) -def _python_contents(*, file_path: Path, payload: _Payload) -> str: - read: Callable[[], str] = { - True: lambda: file_path.read_text(encoding="utf-8"), - False: lambda: str(payload[1]), - }[payload[0] == NATIVE_UNREADABLE_TAG] - return read() +def _source_file( + *, project_dir: Path, relative_path: Path, payload: _Payload +) -> DiscoveredSourceFile: + file_path: Path = project_dir / relative_path + contents, loaded = _loaded(payload=payload, file_path=file_path) + return DiscoveredSourceFile( + file_path=file_path, + relative_path=relative_path, + contents=contents, + source_entries=parse_loaded_sources_yml(loaded=loaded, file_path=file_path), + ) -def _native_supported(*, project_dir: Path, file_paths: tuple[Path, ...]) -> bool: - return ( - bool(file_paths) - and native_discovery_supported( - project_dir=project_dir, display_prefix=native_display_prefix(project_dir) - ) - and all( - native_path_text_supported(_relative_text(project_dir=project_dir, file_path=file_path)) - for file_path in file_paths - ) +def _schema_file(*, project_dir: Path, file_path: Path, payload: _Payload) -> DiscoveredSchemaFile: + contents, loaded = _loaded(payload=payload, file_path=file_path) + model_entries, seed_entries = parse_loaded_schema_yml(loaded=loaded, file_path=file_path) + return DiscoveredSchemaFile( + file_path=file_path, + relative_path=project_relative_path(path=file_path, project_dir=project_dir), + contents=contents, + model_entries=model_entries, + seed_entries=seed_entries, ) +def _loaded(*, payload: _Payload, file_path: Path) -> tuple[str, object]: + error: Exception | None = native_payload_error(payload=payload, file_path=file_path) + if error is not None: + raise error + _tag, contents, loaded = payload + return str(contents), loaded + + def _relative_text(*, project_dir: Path, file_path: Path) -> str: return project_relative_path(path=file_path, project_dir=project_dir).as_posix() def _native_payloads( - *, project_dir: Path, file_paths: tuple[Path, ...] + *, project_dir: Path, file_paths: tuple[Path, ...], kind: str ) -> Iterator[tuple[Path, _Payload]]: """Load the files natively in batches bounded by size, releasing each file once used.""" + if not file_paths: + return tree: _native.NativeProjectTree = native_project_tree(project_dir) + request: dict[str, object] = { + "project_dir": str(project_dir), + "display_prefix": native_display_prefix(project_dir), + "kind": kind, + } for batch in _batches(file_paths=file_paths): - payloads: list[object] | None = _native.load_yaml_files( - [_relative_text(project_dir=project_dir, file_path=path) for path in batch], tree + relative_texts: list[str] = [ + _relative_text(project_dir=project_dir, file_path=path) for path in batch + ] + unnamed: list[_Payload] = [ + native_unreadable_path_payload(project_dir=project_dir, relative_text=text) + for text in relative_texts + ] + payloads: list[_Payload] = native_collection( + result=_native.load_yaml_files( + request, + [ + text + for text, failure in zip(relative_texts, unnamed, strict=True) + if not failure + ], + tree, + ), + project_dir=project_dir, ) seed_snapshot_listings(project_dir=project_dir, tree=tree) - pending: list[_Payload] = ( - [(NATIVE_UNREADABLE_TAG,)] * len(batch) - if payloads is None - else [cast(_Payload, payload) for payload in reversed(payloads)] - ) - del payloads - for file_path in batch: - yield file_path, pending.pop() + payloads.reverse() + for file_path, failure in zip(batch, unnamed, strict=True): + yield file_path, failure or payloads.pop() def _batches(*, file_paths: tuple[Path, ...]) -> Iterator[tuple[Path, ...]]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 9a44737f44..c69a97c45b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -10,10 +10,7 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery.constants import ( - REMOVED_SQL_MODEL_HEADER_KEYS, - SQL_MODEL_HEADER_KEYS, STATEMENT_HEADER_BODY_PATTERN, ) from sqlbuild.compiler.discovery.exceptions import ( @@ -130,90 +127,6 @@ def match_model_header(contents: str) -> ModelHeaderMatch | None: ) -def match_model_headers(contents: list[str]) -> list[ModelHeaderMatch | None]: - """Match many MODEL(...) headers natively with the same semantics as ``match_model_header``.""" - - return [ - None - if offsets is None - else ModelHeaderMatch( - contents=value, - header_start=offsets[0], - header_end=offsets[1], - sql_start=offsets[2], - ) - for value, offsets in zip(contents, _native.match_model_headers(contents), strict=True) - ] - - -def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: - """Parse a raw SQL model file into header values and SQL body.""" - - return parse_matched_model_sql(header_match=match_model_header(contents), file_path=file_path) - - -def parse_matched_model_sql( - *, header_match: ModelHeaderMatch | None, file_path: Path -) -> tuple[dict[str, object], str]: - """Parse header values and SQL body from one precomputed MODEL header match.""" - - if header_match is None: - raise ModelSqlParseError( - f"SQL model '{file_path}' must start with a MODEL(...) header as the first " - "non-whitespace content" - ) - - header_values: dict[str, object] = parse_header_values( - header=header_match.header, - file_path=file_path, - statement_name="MODEL", - ) - removed: list[str] = sorted(REMOVED_SQL_MODEL_HEADER_KEYS.intersection(header_values)) - if removed: - raise ModelSqlParseError( - f"MODEL() option(s) {', '.join(removed)} in '{file_path}' were removed with virtual " - "environments; projects run in direct mode" - ) - reject_unsupported_header_keys( - header_values=header_values, - supported_keys=SQL_MODEL_HEADER_KEYS, - statement="MODEL()", - header=header_match.header, - header_line=header_match.contents.count("\n", 0, header_match.header_start) + 1, - file_path=file_path, - error_class=ModelSqlParseError, - ) - query: str = header_match.sql.strip() - if not query: - raise ModelSqlParseError(f"SQL model '{file_path}' must contain SQL after MODEL(...)") - return header_values, query - - -def model_header_column_locations( - *, contents: str, relative_path: Path -) -> dict[str, SourceLocation]: - """Return authored locations for MODEL(columns) declarations.""" - - return matched_model_header_column_locations( - contents=contents, header_match=match_model_header(contents), relative_path=relative_path - ) - - -def matched_model_header_column_locations( - *, contents: str, header_match: ModelHeaderMatch | None, relative_path: Path -) -> dict[str, SourceLocation]: - """Return authored MODEL(columns) locations from one precomputed header match.""" - - if header_match is None: - return {} - return header_column_locations( - contents=contents, - header=header_match.header, - header_start=header_match.header_start, - relative_path=relative_path, - ) - - def model_header_columns_span( *, contents: str ) -> tuple[int, int, dict[str, ModelHeaderColumnSpan]] | None: @@ -720,20 +633,6 @@ def _header_column_relative_locations( return tuple(locations) -def prepare_model_file_headers(contents: list[str]) -> None: - """Batch tokenization for syntactically recognizable MODEL file headers.""" - - prepare_matched_model_file_headers([match_model_header(value) for value in contents]) - - -def prepare_matched_model_file_headers(header_matches: list[ModelHeaderMatch | None]) -> None: - """Batch tokenization for precomputed MODEL header matches.""" - - prepare_model_header_tokens( - [header_match.header for header_match in header_matches if header_match is not None] - ) - - def project_native_header_values(values: dict[str, object]) -> dict[str, object]: """Project native header parser values, with their marker tuples, to Python header values.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py index 5ac51932c9..4037b9909a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/scenarios.py @@ -2,54 +2,18 @@ from __future__ import annotations -import re -from inspect import cleandoc from pathlib import Path -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIO_HEADER_KEYS, SQL_SCENARIOS_OWNERSHIP_ROOT, - STATEMENT_HEADER_BODY_PATTERN, ) from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile -_SCENARIO_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*SCENARIO\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", - re.DOTALL, -) _SCENARIO_DESCRIPTION_HEADER_KEY: str = "description" _SCENARIO_TAGS_HEADER_KEY: str = "tags" -def parse_sql_scenario_file( - *, contents: str, file_path: Path, relative_path: Path -) -> DiscoveredSqlScenarioFile: - """Parse one SQL-native scenario file.""" - - header_match: re.Match[str] | None = _SCENARIO_HEADER_PATTERN.match(contents) - if header_match is None: - raise SqlScenarioParseError( - f"SQL scenario '{file_path}' must start with a SCENARIO() header as the first " - "non-whitespace content" - ) - - header_values: dict[str, object] = _parse_scenario_header( - header=header_match.group("header"), - header_line=contents.count("\n", 0, header_match.start("header")) + 1, - file_path=file_path, - ) - return build_sql_scenario_file( - header_values=header_values, - sql_body=cleandoc(header_match.group("sql")), - contents=contents, - file_path=file_path, - relative_path=relative_path, - ) - - def build_sql_scenario_file( *, header_values: dict[str, object], @@ -82,23 +46,3 @@ def build_sql_scenario_file( name=file_path.stem, ownership_root=Path(SQL_SCENARIOS_OWNERSHIP_ROOT), ) - - -def _parse_scenario_header(*, header: str, header_line: int, file_path: Path) -> dict[str, object]: - parsed_header: dict[str, object] = parse_header_values( - header=header, - file_path=file_path, - statement_name="SCENARIO", - error_class=SqlScenarioParseError, - ) - - reject_unsupported_header_keys( - header_values=parsed_header, - supported_keys=SQL_SCENARIO_HEADER_KEYS, - statement="SCENARIO()", - header=header, - header_line=header_line, - file_path=file_path, - error_class=SqlScenarioParseError, - ) - return parsed_header diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index 102732d597..c1a66d0748 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -5,7 +5,6 @@ import re from datetime import date, datetime from functools import lru_cache -from inspect import cleandoc from pathlib import Path from typing import cast @@ -18,15 +17,8 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - parse_header_values, - prepare_model_header_tokens, -) from sqlbuild.compiler.discovery.constants import ( SQL_ANALYSIS_CONFIG_KEY, - SQL_TEST_HEADER_KEYS, - STATEMENT_HEADER_BODY_PATTERN, ) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.models import ( @@ -45,14 +37,6 @@ from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -_TEST_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", - re.DOTALL, -) -_TEST_HEADER_ONLY_PATTERN: re.Pattern[str] = re.compile( - r"^\s*TEST\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*", - re.DOTALL | re.MULTILINE, -) _MODE_BY_ACTUAL_CTE_NAME: dict[str, SqlTestMode] = { MACRO_ACTUAL_TEST_CTE_NAME: SqlTestMode.MACRO, UDF_ACTUAL_TEST_CTE_NAME: SqlTestMode.UDF, @@ -94,47 +78,6 @@ _IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -def prepare_sql_test_file_headers(contents_batch: list[str]) -> None: - """Batch cache TEST header parsing before projecting individual files.""" - - headers: list[str] = [] - for contents in contents_batch: - headers.extend( - match.group("header") for match in _TEST_HEADER_ONLY_PATTERN.finditer(contents) - ) - prepare_model_header_tokens(headers) - - -def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: - """Parse one SQL-native test file into one or more raw TEST(...) blocks.""" - - raw_test_blocks: tuple[tuple[str, int], ...] = _split_sql_test_blocks( - file_path=file_path, contents=contents - ) - if not raw_test_blocks: - raise SqlTestParseError( - f"SQL test '{file_path}' must start with a TEST() header as the first " - "non-whitespace content" - ) - - discovered_blocks: list[DiscoveredSqlTestBlock] = [] - test_index: int - raw_test_block: str - block_line: int - for test_index, (raw_test_block, block_line) in enumerate(raw_test_blocks, start=1): - discovered_blocks.append( - _parse_single_sql_test_block( - file_path=file_path, - raw_test_block=raw_test_block, - block_line=block_line, - test_index=test_index, - ) - ) - - validate_sql_test_names(file_path=file_path, blocks=tuple(discovered_blocks)) - return tuple(discovered_blocks) - - def omitted_ceremonial_select_offset_impl(*, sql: str, syntax: SqlLexicalSyntax) -> int | None: """Return the offset after a body's last top-level `)` when only a terminator follows it.""" @@ -236,56 +179,6 @@ def _infer_sql_test_mode(*, sql_body: str, file_path: Path) -> SqlTestMode: return next(iter(inferred_modes), DEFAULT_SQL_TEST_MODE) -def _split_sql_test_blocks(*, file_path: Path, contents: str) -> tuple[tuple[str, int], ...]: - matches: tuple[re.Match[str], ...] = tuple(_TEST_HEADER_ONLY_PATTERN.finditer(contents)) - if not matches: - return () - if contents[: matches[0].start()].strip(): - raise SqlTestParseError( - f"SQL test '{file_path}' must start with a TEST() header as the first " - "non-whitespace content" - ) - - raw_blocks: list[tuple[str, int]] = [] - match_index: int - match: re.Match[str] - for match_index, match in enumerate(matches): - next_start: int = ( - matches[match_index + 1].start() if match_index + 1 < len(matches) else len(contents) - ) - raw_block: str = contents[match.start() : next_start] - block_start: int = match.start() + len(raw_block) - len(raw_block.lstrip()) - raw_blocks.append((raw_block.strip(), contents.count("\n", 0, block_start) + 1)) - return tuple(raw_blocks) - - -def _parse_single_sql_test_block( - *, - file_path: Path, - raw_test_block: str, - block_line: int, - test_index: int, -) -> DiscoveredSqlTestBlock: - header_match: re.Match[str] | None = _TEST_HEADER_PATTERN.match(raw_test_block) - if header_match is None: - raise SqlTestParseError( - f"SQL test '{file_path}' must start with a TEST() header as the first " - "non-whitespace content" - ) - - header_values: dict[str, object] = _parse_test_header( - header=header_match.group("header"), - header_line=block_line + raw_test_block.count("\n", 0, header_match.start("header")), - file_path=file_path, - ) - return build_sql_test_block( - header_values=header_values, - sql_body=cleandoc(header_match.group("sql")), - file_path=file_path, - test_index=test_index, - ) - - def build_sql_test_block( *, header_values: dict[str, object], sql_body: str, file_path: Path, test_index: int ) -> DiscoveredSqlTestBlock: @@ -355,26 +248,6 @@ def _parse_test_cursor_bound( ) -def _parse_test_header(*, header: str, header_line: int, file_path: Path) -> dict[str, object]: - parsed_header: dict[str, object] = parse_header_values( - header=header, - file_path=file_path, - statement_name="TEST", - error_class=SqlTestParseError, - ) - - reject_unsupported_header_keys( - header_values=parsed_header, - supported_keys=SQL_TEST_HEADER_KEYS, - statement="TEST()", - header=header, - header_line=header_line, - file_path=file_path, - error_class=SqlTestParseError, - ) - return parsed_header - - def _parse_test_parameters_and_cases( *, header_values: dict[str, object], file_path: Path ) -> tuple[tuple[SqlTestParameterDeclaration, ...], tuple[DiscoveredSqlTestCase, ...]]: diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py index 2ae6db08d8..4bce113bd5 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py @@ -5,9 +5,6 @@ from pathlib import Path from typing import cast -import yaml -from yaml import YAMLError - from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.authored_values.main._optional_bool import optional_bool from sqlbuild.compiler.authored_values.main._optional_mapping import optional_mapping @@ -21,7 +18,10 @@ from sqlbuild.compiler.discovery._helpers.validation.supported_keys import ( reject_unknown_mapping_keys, ) -from sqlbuild.compiler.discovery.constants import NOT_NULL_AUDIT_NAME, SEEDS_DIRECTORY_NAME +from sqlbuild.compiler.discovery.constants import ( + NOT_NULL_AUDIT_NAME, + SEEDS_DIRECTORY_NAME, +) from sqlbuild.compiler.discovery.exceptions import SchemaParseError from sqlbuild.spec.contracts.models import ( SchemaAuditInstance, @@ -69,18 +69,6 @@ ) -def parse_schema_yml( - *, - contents: str, - file_path: Path, -) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: - """Parse one schema.yml file into raw model and seed metadata.""" - - return parse_loaded_schema_yml( - loaded=_load_schema_yaml(contents=contents, file_path=file_path), file_path=file_path - ) - - def parse_loaded_schema_yml( *, loaded: object, file_path: Path ) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: @@ -93,14 +81,6 @@ def parse_loaded_schema_yml( ) -def _load_schema_yaml(*, contents: str, file_path: Path) -> object: - try: - loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - return yaml.load(contents, Loader=loader) - except YAMLError as error: - raise SchemaParseError(f"{file_path} contains invalid YAML: {error}") from error - - def _schema_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: payload: object = loaded if payload is None: diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py index 3bdd09915a..bf1ad163bd 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/sources.py @@ -5,9 +5,6 @@ from pathlib import Path from typing import cast -import yaml -from yaml import YAMLError - from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.authored_values.main._optional_bool import optional_bool from sqlbuild.compiler.authored_values.main._optional_mapping import optional_mapping @@ -112,14 +109,6 @@ ) -def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: - """Parse one sources/*.yml file into raw source declarations.""" - - return parse_loaded_sources_yml( - loaded=_load_sources_yaml(contents=contents, file_path=file_path), file_path=file_path - ) - - def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[SourceEntry, ...]: """Parse the document one sources/*.yml file loads to into raw source declarations.""" @@ -143,14 +132,6 @@ def parse_loaded_sources_yml(*, loaded: object, file_path: Path) -> tuple[Source return tuple(parsed_sources) -def _load_sources_yaml(*, contents: str, file_path: Path) -> object: - try: - loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) - return yaml.load(contents, Loader=loader) - except YAMLError as error: - raise SourceParseError(f"{file_path} contains invalid YAML: {error}") from error - - def _sources_mapping(*, loaded: object, file_path: Path) -> dict[str, object]: payload: object = loaded if payload is None: diff --git a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py index 0f5878b792..63b49da16b 100644 --- a/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py +++ b/src/sqlbuild/compiler/discovery/classes/selected_contract_input_discoverer.py @@ -4,18 +4,20 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_model_files, - discover_model_schema_files, - discover_schema_files, - discover_source_files, - discover_test_files, +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files +from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files +from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( + discover_native_schema_files, + discover_native_source_files, ) from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, ) +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig class SelectedContractInputDiscoverer: @@ -30,21 +32,24 @@ def discover( ) -> DiscoveredProjectInputs: """Build selected contract inputs without importing Python runtime resources.""" - return DiscoveredProjectInputs( - project_config=load_project_config(project_dir=project_dir), - local_config=load_local_config(project_dir=project_dir), - project_dir=project_dir, - model_files=discover_model_files( + project_config: ProjectConfig = load_project_config(project_dir=project_dir) + local_config: LocalConfig = load_local_config(project_dir=project_dir) + with DirectorySnapshot.scope(project_dir=project_dir): + return DiscoveredProjectInputs( + project_config=project_config, + local_config=local_config, project_dir=project_dir, - extract_implicit_alias_columns=False, - extract_output_column_locations=False, - selected_model_names=referenced_model_names, - ), - model_schema_files=discover_model_schema_files(project_dir=project_dir), - schema_files=discover_schema_files(project_dir=project_dir), - source_files=discover_source_files(project_dir=project_dir), - test_files=discover_test_files( - project_dir=project_dir, - selected_paths=selected_test_paths, - ), - ) + model_files=discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=False, + extract_output_column_locations=False, + selected_model_names=referenced_model_names, + ), + model_schema_files=discover_model_schema_files(project_dir=project_dir), + schema_files=discover_native_schema_files(project_dir=project_dir), + source_files=discover_native_source_files(project_dir=project_dir), + test_files=discover_native_test_files( + project_dir=project_dir, + selected_paths=selected_test_paths, + ), + ) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 2b02d9f48b..4435d6ebf3 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -180,11 +180,10 @@ r"|\)(?!\s*;))*+)" ) -DISCOVERY_FACT_CACHE_NAMESPACE: str = "discovery" -DISCOVERY_FACT_CACHE_ALGORITHM: str = "discovered-files-v1" -DISCOVERY_SOURCE_FACT_KIND: str = "source" -DISCOVERY_SQL_TEST_FACT_KIND: str = "sql_test" NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" -NATIVE_LOADED_TAG: str = "ok" +NATIVE_UNLISTABLE_TAG: str = "unlistable" +NATIVE_SOURCE_YAML_KIND: str = "source" +NATIVE_SCHEMA_YAML_KIND: str = "schema" +NATIVE_SUPPORTED_PYTHON_VERSIONS: str = "3.12, 3.13 and 3.14" NATIVE_YAML_BATCH_BYTES: int = 4 * 1024 * 1024 diff --git a/src/sqlbuild/compiler/discovery/exceptions.py b/src/sqlbuild/compiler/discovery/exceptions.py index 7779d906fb..f9522ba124 100644 --- a/src/sqlbuild/compiler/discovery/exceptions.py +++ b/src/sqlbuild/compiler/discovery/exceptions.py @@ -109,3 +109,15 @@ class ProjectPythonPathError(DiscoveryError): """Raised when project-owned Python lives outside a supported extension root.""" code: str = "D016" + + +class ProjectPathError(DiscoveryError): + """Raised when an authored project path cannot be read as valid UTF-8.""" + + code: str = "D016" + + +class UnsupportedPythonError(DiscoveryError): + """Raised when discovery runs on a Python whose text semantics this release does not know.""" + + code: str = "D017" diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py index 5925541c97..4784e2421d 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -4,31 +4,32 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_model_files, - discover_model_schema_files, -) +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_schema_files +from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config +from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault -from sqlbuild.spec.contracts.models import LocalConfig +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: """Load project config, model files and model schemas without importing project Python.""" - return DiscoveredProjectInputs( - project_config=load_project_config(project_dir=project_dir), - local_config=LocalConfig(), - model_files=discover_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=False, - extract_output_column_locations=False, - on_fault=_skip_unreadable_file, - ), - model_schema_files=discover_model_schema_files( - project_dir=project_dir, on_fault=_skip_unreadable_file - ), - ) + project_config: ProjectConfig = load_project_config(project_dir=project_dir) + with DirectorySnapshot.scope(project_dir=project_dir): + return DiscoveredProjectInputs( + project_config=project_config, + local_config=LocalConfig(), + model_files=discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=False, + extract_output_column_locations=False, + on_fault=_skip_unreadable_file, + ), + model_schema_files=discover_model_schema_files( + project_dir=project_dir, on_fault=_skip_unreadable_file + ), + ) def _skip_unreadable_file(fault: DiscoveryFileFault) -> None: diff --git a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py b/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py index 6c366a9231..a91f696df2 100644 --- a/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py +++ b/src/sqlbuild/compiler/discovery/main/_parse_sql_test_file.py @@ -4,11 +4,13 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.sql.tests import parse_sql_test_file as _parse +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + parse_native_sql_test_contents, +) from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: """Parse current SQL test contents into ordered test blocks.""" - return _parse(contents=contents, file_path=file_path) + return parse_native_sql_test_contents(contents=contents, file_path=file_path) diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index 394aaa28bd..560e3ed11b 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -8,7 +8,6 @@ from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( build_discovered_project_inputs, ) -from sqlbuild.compiler.discovery._helpers.filesystem.cache_root import discovery_cache_root from sqlbuild.compiler.discovery._helpers.filesystem.python_paths import ( validate_project_python_paths, ) @@ -22,17 +21,11 @@ validate_local_sql_analysis_policy, validate_target_warehouses_adapter, ) -from sqlbuild.compiler.discovery.constants import ( - DISCOVERY_FACT_CACHE_ALGORITHM, - DISCOVERY_FACT_CACHE_NAMESPACE, - SQL_ANALYSIS_SETTING_KEY, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryCacheRequest +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.discovery.types import DeclarationFilesReuse -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier -from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled -from sqlbuild.compiler.frontier.types import CompilerStage, NativeStage +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -42,7 +35,6 @@ def discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None = None, extract_output_column_locations: bool = True, - cache_request: DiscoveryCacheRequest | None = None, declaration_reuse: DeclarationFilesReuse | None = None, ) -> DiscoveredProjectInputs: """Load all raw project inputs from disk before semantic resolution.""" @@ -55,22 +47,7 @@ def discover_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, declaration_reuse=declaration_reuse, - native=False, - ), - native_stage=( - partial( - _discover_project_inputs, - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, - declaration_reuse=declaration_reuse, - native=True, - ) - if native_stage_enabled(NativeStage.DISCOVERY) - else None ), ) @@ -80,18 +57,14 @@ def _discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - cache_request: DiscoveryCacheRequest | None, declaration_reuse: DeclarationFilesReuse | None, - native: bool, ) -> DiscoveredProjectInputs: with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): return _assemble_discovered_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, declaration_reuse=declaration_reuse, - native=native, ) @@ -100,9 +73,7 @@ def _assemble_discovered_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - cache_request: DiscoveryCacheRequest | None, declaration_reuse: DeclarationFilesReuse | None, - native: bool, ) -> DiscoveredProjectInputs: project_config: ProjectConfig = load_project_config(project_dir=project_dir) local_config: LocalConfig = load_local_config(project_dir=project_dir) @@ -125,26 +96,14 @@ def _assemble_discovered_project_inputs( else project_config.settings.sql_analysis ) ) - with FactCacheStore( - root=discovery_cache_root( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - cache_request=cache_request, - ), - namespace=DISCOVERY_FACT_CACHE_NAMESPACE, - algorithm=DISCOVERY_FACT_CACHE_ALGORITHM, - ) as fact_cache: - discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( - project_dir=project_dir, - project_config=project_config, - local_config=local_config, - sql_analysis_enabled=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - fact_cache=fact_cache, - declaration_reuse=declaration_reuse, - native=native, - ) + discovered_inputs: DiscoveredProjectInputs = build_discovered_project_inputs( + project_dir=project_dir, + project_config=project_config, + local_config=local_config, + sql_analysis_enabled=sql_analysis_enabled, + extract_output_column_locations=extract_output_column_locations, + declaration_reuse=declaration_reuse, + ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( configure_discovered_event_exporters, diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index d2d289568f..5434a8446e 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -618,14 +618,6 @@ class DiscoveredPythonNodeFunctions: audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DiscoveryCacheRequest: - """Compile-cache controls that let discovery reuse exact per-file parse facts.""" - - selected_target: str | None = None - no_cache: bool = False - - @dataclass(frozen=True) class DiscoveredDeclarationFiles: """Project files discovery parses without importing any project Python.""" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 980b716ff4..7e82f1e95f 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -21,6 +21,7 @@ type NativeLocation = tuple[str, int, int, int, int] type NativeListing = tuple[str, list[tuple[str, bool, bool]]] type NativeDeclarationFact = tuple[str, str, str, str, str | None, str] +type NativeFiles = list[tuple[str, tuple[object, ...]]] class LoaderConnectionMode(StrEnum): diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index b0694a6d7b..8f07f996bd 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -13,7 +13,10 @@ emitted_failure_codes, render_error_codes, ) -from scripts.compiler_differential.constants import RENDER_UNREACHABLE_CODES +from scripts.compiler_differential.constants import ( + DISCOVERY_UNREACHABLE_CODES, + RENDER_UNREACHABLE_CODES, +) from scripts.compiler_differential.main.failure_cases import failure_cases from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase @@ -37,7 +40,8 @@ for case in failure_cases() if case.expected_warning_code is not None }, - expected_discovery_codes=discovery_error_codes(), + expected_discovery_codes=discovery_error_codes() + - frozenset(DISCOVERY_UNREACHABLE_CODES), expected_render_codes=render_error_codes() - frozenset(RENDER_UNREACHABLE_CODES), unreachable_render_codes=frozenset(RENDER_UNREACHABLE_CODES), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py index ea616428b6..31d9059f0a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -23,3 +23,11 @@ class UnicodeEmptyFixtureTestCase: description: str status: str expected_added_findings: int = 0 + + +@dataclass(frozen=True) +class LegacyCodePageOutputTestCase: + description: str + code_page: str + expected_summary: str + expected_json_command: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py new file mode 100644 index 0000000000..7c1363175b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from sqlbuild.cli.commands.main.workspace._playground import run_playground +from sqlbuild.cli.commands.models import PlaygroundCommandRequest +from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( + LegacyCodePageOutputTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + LegacyCodePageOutputTestCase( + description="cp1252 streams still get the UTF-8 summary and valid JSON", + code_page="cp1252", + expected_summary="\u2713 Project compiled", + expected_json_command="compile", + ), + ], + ids=lambda case: case.description, +) +def test_given_legacy_code_page_streams_when_compiling_then_output_is_utf8( + test_case: LegacyCodePageOutputTestCase, tmp_path: Path +) -> None: + assert run_playground(PlaygroundCommandRequest(project_dir=tmp_path, target_path="shop")) == 0 + environment: dict[str, str] = {"PYTHONIOENCODING": test_case.code_page} + + summary: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=tmp_path / "shop", env=environment + ) + machine: subprocess.CompletedProcess[str] = run_sqb( + command=("compile", "--json"), project_dir=tmp_path / "shop", env=environment + ) + + assert ( + summary.returncode, + test_case.expected_summary in summary.stdout, + machine.returncode, + json.loads(machine.stdout).get("command"), + ) == (0, True, 0, test_case.expected_json_command), summary.stderr + machine.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 575c9be04b..444e0ba56d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -13,6 +13,7 @@ import duckdb import pytest +from sqlbuild._native import NativeProjectTree from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -28,12 +29,11 @@ from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.compile._helpers.assembly import source_bindings as source_bindings_module from sqlbuild.compiler.compile.models import PolyglotAnalysisResult -from sqlbuild.compiler.discovery._helpers.filesystem import ( - model_files as discovery_model_files_module, +from sqlbuild.compiler.discovery._helpers.native import ( + model_files as native_model_files_module, ) from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact -from sqlbuild.spec.contracts.models import SourceLocation from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) @@ -651,22 +651,18 @@ def recording(**kwargs: Any) -> PolyglotAnalysisResult: return analysed -def record_eager_output_column_scans(monkeypatch: pytest.MonkeyPatch) -> list[Path]: - """Record every model file whose output columns discovery locates eagerly.""" +def record_eager_output_column_scans(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: + """Record every native model discovery request that locates output columns eagerly.""" - scanned: list[Path] = [] - original: Callable[..., dict[str, SourceLocation]] = ( - discovery_model_files_module.matched_model_output_column_locations - ) + eager: list[dict[str, object]] = [] + original: Callable[..., object] = native_model_files_module._native.discover_model_files - def recording(**kwargs: Any) -> dict[str, SourceLocation]: - scanned.append(cast(Path, kwargs["relative_path"])) - return original(**kwargs) + def recording(request: dict[str, object], tree: NativeProjectTree) -> object: + eager.extend(filter(lambda item: item["extract_output_column_locations"], (request,))) + return original(request, tree) - monkeypatch.setattr( - discovery_model_files_module, "matched_model_output_column_locations", recording - ) - return scanned + monkeypatch.setattr(native_model_files_module._native, "discover_model_files", recording) + return eager def write_repeated_json_parse_project( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 13e71a9007..53c3a4c471 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -57,7 +57,7 @@ def test_given_project_when_running_command_then_discovery_skips_eager_column_lo 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), ) - scanned: list[Path] = record_eager_output_column_scans(monkeypatch) + scanned: list[dict[str, object]] = record_eager_output_column_scans(monkeypatch) _ = capsys.readouterr() exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 9157d3e3ea..62400192f1 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -47,26 +47,6 @@ class LargeDocumentOracleTestCase: expected_maximum_seconds: float = 2.0 -@dataclass(frozen=True) -class ModelDiscoveryParityTestCase: - """Authored model files discovered by both compiler engines.""" - - description: str - files: tuple[tuple[str, bytes], ...] - directories: tuple[str, ...] = () - expected_identical: bool = True - - -@dataclass(frozen=True) -class GeneratedModelParityTestCase: - """Seeded model files discovered by both compiler engines, one file at a time.""" - - description: str - seed: int - case_count: int - expected_mismatches: tuple[tuple[object, object, object], ...] = () - - @dataclass(frozen=True) class EngineSwitchParityTestCase: """A project discovered through `discover_project_inputs` under each engine.""" @@ -77,12 +57,14 @@ class EngineSwitchParityTestCase: @dataclass(frozen=True) -class NativeDeferralTestCase: - """A project and runtime under which native model discovery must or must not run.""" +class NativeRuntimeTestCase: + """A project and runtime under which native model discovery runs or fails clearly.""" description: str project_name: str - expected_native_calls: int + expected_models: int + expected_error: str = "NoneType" + expected_message: str = "None" unidata_version: str = field(default_factory=lambda: unicodedata.unidata_version) python_version: tuple[int, int] = field( default_factory=lambda: (sys.version_info[0], sys.version_info[1]) @@ -100,55 +82,83 @@ class SharedSnapshotTestCase: @dataclass(frozen=True) -class GeneratedLayoutParityTestCase: - """Seeded declaration layouts whose native validation and facts must match Python's.""" +class NativeYamlLoadTestCase: + """Source files whose native load outcomes and loaded values are checked.""" description: str - seed: int - count: int - expected_mismatches: tuple[str, ...] = () - expected_minimum_valid: int = 0 - expected_minimum_invalid: int = 0 + files: tuple[tuple[str, bytes], ...] + expected_native_tags: tuple[str, ...] @dataclass(frozen=True) -class GeneratedSqlTestParityTestCase: - """Seeded SQL test and scenario files discovered by both compiler engines.""" +class ReadErrorOracleTestCase: + """Seeded byte strings whose native read error must equal Python's `read_text` error.""" description: str seed: int case_count: int - expected_mismatches: tuple[tuple[object, object, object], ...] = () - expected_minimum_parsed: int = 0 - expected_minimum_failed: int = 0 + expected_minimum_failures: int @dataclass(frozen=True) -class GeneratedYamlFileParityTestCase: - """Seeded source and seed declaration files discovered by both compiler engines.""" +class UnreadableFileTestCase: + """One unreadable authored file that compile reports with Python's own read error.""" description: str - seed: int - case_count: int - expected_mismatches: tuple[tuple[object, object, object], ...] = () - expected_minimum_parsed: int = 0 - expected_minimum_failed: int = 0 + relative_path: str + data: bytes + expected_error: str + expected_message: str @dataclass(frozen=True) -class NativeYamlLoadTestCase: - """Source files whose native load outcome and engine parity are both checked.""" +class AuthoredFileFailureTestCase: + """Authored files that discovery reads, or rejects with one clear error.""" description: str files: tuple[tuple[str, bytes], ...] - expected_native_tags: tuple[str, ...] + expected_error: str = "NoneType" + expected_message: str = "None" + expected_help: bool = False @dataclass(frozen=True) -class FactCacheFallbackTestCase: - """A runtime under which native source and test discovery must use the fact cache or not.""" +class UndecodablePathToleranceTestCase: + """A project with names that are not UTF-8, read by a caller that tolerates bad files.""" description: str - unidata_version: str - expected_same_keys_as_python: bool - expected_native_keys: int + reader: str + files: tuple[tuple[str, bytes], ...] + expected_models: tuple[str, ...] + expected_sources: tuple[str, ...] + expected_faults: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True) +class UnsupportedPythonCommandTestCase: + """A command run on a Python this release does not support.""" + + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_error: str + + +@dataclass(frozen=True) +class NativeOsErrorTestCase: + """An operating-system read payload and the Python error it becomes.""" + + description: str + payload: tuple[object, ...] + expected_type: str + expected_fields: tuple[object, ...] + + +@dataclass(frozen=True) +class CollidingNamesTestCase: + """Files whose names are not UTF-8 but share a lossy spelling, each keeping its own identity.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_enums: tuple[tuple[str, tuple[str, ...]], ...] + expected_models: tuple[str, ...] = ("models/orders.sql",) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index c894609cb0..860eca1d94 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -18,46 +18,25 @@ import yaml from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_constant_files, - discover_enum_files, - discover_macro_files, - discover_model_files, - discover_scenario_files, - discover_schema_files, - discover_source_files, - discover_test_files, +from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( + build_tolerant_scope_discovery, ) -from sqlbuild.compiler.discovery._helpers.filesystem.named_declarations import ( - named_declaration_roots, - remember_declaration_groups, -) -from sqlbuild.compiler.discovery._helpers.native.model_files import ( - discover_native_model_files, -) -from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( - discover_native_scenario_files, - discover_native_test_files, +from sqlbuild.compiler.discovery._helpers.native.payloads import native_payload_error +from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( + SelectedContractInputDiscoverer, ) -from sqlbuild.compiler.discovery._helpers.native.yaml_files import ( - discover_native_schema_files, - discover_native_source_files, +from sqlbuild.compiler.discovery.main._model_description_inputs import ( + discover_model_description_inputs, ) -from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config -from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( - DiscoveredConstantFile, DiscoveredEnumFile, - DiscoveredMacroFile, - DiscoveredSqlModelFile, + DiscoveredProjectInputs, + TolerantScopeDiscovery, ) -from sqlbuild.compiler.discovery.types import NativeDeclarationFact -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.compiler.scopes.constants import NAMED_DECLARATION_KINDS from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text @@ -67,6 +46,9 @@ DEFERRED: dict[str, str] = {"error": "deferred to Python"} LOADERS_DISAGREE: dict[str, str] = {"error": "PyYAML and LibYAML disagree"} UNCHECKED_PYTHON_OUTCOMES: tuple[dict[str, str], ...] = (PYTHON_ERROR, LOADERS_DISAGREE) +UNDECODABLE_BYTES: tuple[int, ...] = ( + 0x80, 0x9F, 0xA0, 0xBF, 0xC0, 0xC1, 0xC2, 0xE0, 0xE2, 0xED, 0xF0, 0xF4, 0xF5, 0xFF, +) # fmt: skip YAML_LINE_BREAKS: dict[int, None] = {0x85: None, 0x2028: None, 0x2029: None} SCALAR_FRAGMENTS: tuple[str, ...] = ( "0", "1", "7", "8", "9", "00", "12", "59", "60", "_", ".", ":", "e", "E", "e+", "E-", @@ -636,250 +618,6 @@ def harmful_mismatches( PROJECT_CONFIG: str = 'name = "orders"\nadapter = "duckdb"\n' -HEADER_ENTRIES: tuple[str, ...] = ( - "materialized table", - "description 'Orders by customer'", - "description \"é ☕ 'quoted'\"", - "tags [core, finance]", - "unique_key [order_id]", - 'columns (order_id (type INTEGER), "Total" (type DECIMAL(10, 2)))', - "columns (\n customer_id (type BIGINT, audits [not_null]),\n )", - "enums (_status [OPEN, CLOSED])", - "enums (status [open])", - "constants (_limit_rows 10, _ratio 1.5)", - 'pre_hooks ["SELECT 1"]', - "tagz [a]", - "materialised table", - "run_despite_unchanged true", - "materialized: table", - "description 'unterminated", - "/* note */ colums (id (type INT))", - "-- trailing comment\n audits [unique]", -) -SELECT_ITEMS: tuple[str, ...] = ( - "order_id", - "o.customer_id", - "total AS amount", - "total as Amount", - "total aſ amount", - "sum(total) total_sum", - "COUNT(*)", - '"Quoted Name"', - 'o."Ünit"', - "'a,b' AS text_value", - "'it\\'s' AS escaped", - "CASE WHEN a THEN 1 END AS flag", - "(SELECT 1) sub", - "café", - "1 AS ünder", -) -TAILS: tuple[str, ...] = ( - "", - " FROM orders o", - " FROM orders\nUNION ALL\nSELECT 1 FROM b", - " FROM orders unıon SELECT 1", - " FROM (SELECT 1 FROM x) t", - " fRoM orders", - "\n-- FROM comment", -) -SEPARATORS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", " ", "\u3000", "\n\n") - - -TEST_HEADER_ENTRIES: tuple[str, ...] = ( - 'name "keeps_orders"', "mode model", 'name "second"', "name ''", "name 1", "mode query", - "mode audit", "mode bogus", "sql_analysis false", "sql_analysis 1", "nme 1", - "parameters (status string), cases (placed (status 'placed'))", - "parameters (amount (type integer, nullable true)), cases (empty (amount null))", - "parameters (amount integer), cases (bad (amount 'x'))", "parameters (amount integer)", - "cursor_start '2024-01-01'", "cursor_end 3", "materialized table", "description (", - "tags [a, b]", "'unterminated", -) # fmt: skip -SCENARIO_HEADER_ENTRIES: tuple[str, ...] = ( - 'description "Orders world"', "tags [north, south]", "description 1", "tags [1]", - "tags north", "name x", "description (", '"unterminated', -) # fmt: skip -STATEMENT_BODIES: tuple[str, ...] = ( - "SELECT 1", "SELECT 2 AS amount", "\n SELECT 3\n FROM orders\n", "", " ", - "\n SELECT 1\n FROM t\n", "\tSELECT 1\n\tFROM\tt", - "SELECT 'TEST (x);' AS label", "SELECT 1\n ", "-- only a comment", "SELECT ')' AS c;", - "\u3000SELECT 1", -) # fmt: skip -STATEMENT_PREFIXES: tuple[str, ...] = ("",) * 10 + (" \n", "\ufeff", "-- lead\n", "\n\n ") - - -def generated_statement_text(*, rng: random.Random, keyword: str, entries: tuple[str, ...]) -> str: - """Return one statement header with random entries, separators and a body.""" - - separator: str = rng.choice(SEPARATORS) - chosen: list[str] = rng.choices( - (rng.sample(entries, k=rng.choice((0, 1, 1, 1, 2))), rng.sample(entries[:2], k=1)), - weights=(3, 2), - )[0] - header: str = ("," + separator).join(chosen) - closing: str = rng.choice((");",) * 8 + (") ;", ")\n;", ")", "));", ");;")) - return ( - f"{rng.choice(('', ' ', ' '))}{keyword}{rng.choice(('', ' '))}({separator}{header}" - f"{separator}{closing}{rng.choice(SEPARATORS)}{rng.choice(STATEMENT_BODIES)}" - ) - - -def generated_test_bytes(*, rng: random.Random) -> bytes: - """Return one SQL test file with one to three blocks, joined at or inside lines.""" - - blocks: list[str] = [ - generated_statement_text(rng=rng, keyword="TEST", entries=TEST_HEADER_ENTRIES) - for _ in range(rng.choice((1, 1, 1, 1, 2, 3))) - ] - joined: str = "".join(rng.choice(("\n", "\n\n", " ", "\r\n")) + block for block in blocks) - return (rng.choice(STATEMENT_PREFIXES) + joined.lstrip("\n\r ")).encode("utf-8") - - -def generated_scenario_bytes(*, rng: random.Random) -> bytes: - """Return one SQL scenario file.""" - - text: str = generated_statement_text( - rng=rng, keyword="SCENARIO", entries=SCENARIO_HEADER_ENTRIES - ) - return (rng.choice(STATEMENT_PREFIXES) + text).encode("utf-8") - - -def sql_test_discovery_outcome(*, project_dir: Path, native: bool) -> object: - """Return the rendered test and scenario files, or the error type, message and help.""" - - discover: Callable[[], tuple[object, object]] = { - True: lambda: ( - discover_native_test_files(project_dir=project_dir), - discover_native_scenario_files(project_dir=project_dir), - ), - False: lambda: ( - discover_test_files(project_dir=project_dir), - discover_scenario_files(project_dir=project_dir), - ), - }[native] - capture: FailureCapture = FailureCapture() - rendered: list[object] = [None] - with DirectorySnapshot.scope(project_dir=project_dir), capture: - rendered[0] = render_stage_capture(discover()) - failure: BaseException | None = capture.failure - return { - True: rendered[0], - False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), - }[failure is None] - - -SOURCE_FIELD_CHOICES: dict[str, tuple[object, ...]] = { - "description": ("Orders feed.", "", 3, None), - "database": ("raw", ""), - "schema": ("shop", None), - "table": ("orders", 7), - "managed": (True, False, "yes"), - "write_strategy": ("append", "merge", "table", "bogus"), - "unique_key": ("order_id", ["order_id", "line"], [], ""), - "cursor_column": ("updated_at",), - "load_batch_size": (100, 0, True), - "type_enforcement": (True, "no"), - "contract": ("enforced", "none", "strict"), - "freshness": ( - {"strategy": "adapter"}, - {"strategy": "column", "column": "updated_at", "type": "timestamp", "lag_tolerance": "2h"}, - {"strategy": "sql", "query": "SELECT 1", "type": "timestamp"}, - {"strategy": "column"}, - ), - "audits": (["not_null"], [{"accepted_values": {"values": [1, 2]}}], "x"), - "freshnes": ("1d",), -} -COLUMN_FIELD_CHOICES: dict[str, tuple[object, ...]] = { - "type": ("INTEGER", "VARCHAR", ""), - "nullable": (True, False, "yes", None), - "description": ("Order id.", 5), - "audits": (["not_null"], ["unique"], []), -} -SCALAR_META_TEMPLATES: tuple[str, ...] = ( - "\n meta: {{flag: {plain}}}", "\n meta:\n seen: {plain}", - "\n meta: &shared{n} {{k: {plain}}}", "\n description: '{single}'", - "\n <<: {{description: '{single}'}}", "\n meta: {{k: {plain}, j: [{plain}]}}", -) # fmt: skip - - -def _random_fields( - *, rng: random.Random, choices: dict[str, tuple[object, ...]] -) -> dict[str, object]: - keys: list[str] = rng.sample(sorted(choices), k=rng.randint(0, 3)) - return { - key: rng.choices(choices[key], weights=(6, *(1,) * (len(choices[key]) - 1)))[0] - for key in keys - } - - -def _generated_column(*, rng: random.Random, names: tuple[str, ...]) -> dict[str, object]: - return {"name": rng.choice(names), **_random_fields(rng=rng, choices=COLUMN_FIELD_CHOICES)} - - -def _generated_entry( - *, - rng: random.Random, - names: tuple[str, ...], - column_names: tuple[str, ...], - fields: dict[str, object], -) -> dict[str, object]: - return { - "name": rng.choice(names), - **fields, - "columns": [ - _generated_column(rng=rng, names=column_names) for _ in range(rng.randint(0, 3)) - ], - "meta": {"owner": random_yaml_value(rng=rng, depth=2)}, - } - - -def generated_source_document(*, rng: random.Random) -> str: - """Return a sources file of random entries in a random dump style, with resolver scalars.""" - - sources: list[object] = [ - _generated_entry( - rng=rng, - names=("orders", "customers", "line_items", ""), - column_names=("order_id", "amount", "status"), - fields=_random_fields(rng=rng, choices=SOURCE_FIELD_CHOICES), - ) - for _ in range(rng.randint(0, 3)) - ] - flow_style: bool | None = rng.choice((None, False, False, True)) - dumped: str = yaml.safe_dump( - {"sources": sources}, - default_flow_style=flow_style, - default_style=rng.choice((None, None, None, '"', "'")), - width=rng.choice((40, 80, 4096)), - allow_unicode=rng.random() < 0.5, - sort_keys=False, - ) - scalar: str = random_scalar(rng=rng) - tail: str = rng.choice(SCALAR_META_TEMPLATES).format( - plain=scalar, single=scalar.replace("'", "''"), n=rng.randint(0, 9) - ) - extra: str = rng.choices((f"- name: extra{tail}\n", "other: [\n"), weights=(8, 1))[0] - appendable: bool = flow_style is False and bool(sources) - return rng.choice(BYTE_ORDER_MARKS) + dumped + {True: extra, False: ""}[appendable] - - -def generated_seed_declaration(*, rng: random.Random) -> str: - """Return a seed declaration file of random entries in a random dump style.""" - - seeds: list[object] = [ - _generated_entry( - rng=rng, - names=("channels", "regions", ""), - column_names=("id", "label"), - fields={"description": rng.choice(("Order channels.", 1))}, - ) - for _ in range(rng.randint(0, 2)) - ] - return yaml.safe_dump( - {"seeds": seeds}, - default_flow_style=rng.choice((None, False, True)), - default_style=rng.choice((None, None, '"')), - sort_keys=False, - ) def native_yaml_tags_and_values( @@ -887,39 +625,20 @@ def native_yaml_tags_and_values( ) -> tuple[list[object], bool]: """Return each file's native tag and whether every natively loaded value equals LibYAML's.""" - payloads: list[object] = ( - _native.load_yaml_files(relative_paths, _native.NativeProjectTree(str(project_dir))) or [] + payloads: list[tuple[object, ...]] = cast( + list[tuple[object, ...]], + _native.load_yaml_files( + {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, + relative_paths, + _native.NativeProjectTree(str(project_dir)), + ), ) - rows: list[tuple[object, ...]] = [cast(tuple[object, ...], payload) for payload in payloads] + rows: list[tuple[object, ...]] = list(payloads) loaded: list[tuple[object, ...]] = list(compress(rows, [row[0] == "ok" for row in rows])) expected: list[object] = [yaml.load(str(row[1]), Loader=yaml.CSafeLoader) for row in loaded] return [row[0] for row in rows], [row[2] for row in loaded] == expected -def yaml_discovery_outcome(*, project_dir: Path, native: bool) -> object: - """Return the rendered source and schema files, or the error type, message and help.""" - - discover: Callable[[], tuple[object, object]] = { - True: lambda: ( - discover_native_source_files(project_dir=project_dir), - discover_native_schema_files(project_dir=project_dir), - ), - False: lambda: ( - discover_source_files(project_dir=project_dir), - discover_schema_files(project_dir=project_dir), - ), - }[native] - capture: FailureCapture = FailureCapture() - rendered: list[object] = [None] - with DirectorySnapshot.scope(project_dir=project_dir), capture: - rendered[0] = render_stage_capture(discover()) - failure: BaseException | None = capture.failure - return { - True: rendered[0], - False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), - }[failure is None] - - class FailureCapture: """Swallow and keep the discovery failure a block raises.""" @@ -931,31 +650,7 @@ def __enter__(self) -> FailureCapture: def __exit__(self, error_type: object, error: BaseException | None, traceback: object) -> bool: self.failure = error - return isinstance(error, OSError | ValueError) - - -def model_discovery_outcome(*, project_dir: Path, native: bool) -> object: - """Return the rendered model files, or the error type, message and help.""" - - discover: Callable[..., tuple[DiscoveredSqlModelFile, ...]] = { - True: discover_native_model_files, - False: discover_model_files, - }[native] - capture: FailureCapture = FailureCapture() - rendered: list[object] = [None] - with DirectorySnapshot.scope(project_dir=project_dir), capture: - rendered[0] = render_stage_capture( - discover( - project_dir=project_dir, - extract_implicit_alias_columns=True, - extract_output_column_locations=True, - ) - ) - failure: BaseException | None = capture.failure - return { - True: rendered[0], - False: (type(failure).__name__, str(failure), getattr(failure, "help", None)), - }[failure is None] + return isinstance(error, OSError | ValueError | RuntimeError) def write_project(*, project_dir: Path, files: tuple[tuple[str, bytes], ...]) -> None: @@ -969,225 +664,132 @@ def write_project(*, project_dir: Path, files: tuple[tuple[str, bytes], ...]) -> _ = path.write_bytes(data) -def generated_model_bytes(*, rng: random.Random) -> bytes: - """Return one model file mixing valid and invalid headers, projections and newlines.""" - - separator: str = rng.choice(SEPARATORS) - entries: list[str] = rng.sample(HEADER_ENTRIES, k=rng.randint(0, 2)) - header: str = ("," + separator + " ").join(entries) - items: list[str] = rng.sample(SELECT_ITEMS, k=rng.randint(1, 4)) - body: str = "SELECT " + ("," + rng.choice(SEPARATORS)).join(items) + rng.choice(TAILS) - prefix: str = rng.choice(("",) * 8 + (" \n", "\ufeff", "-- lead\n")) - text: str = rng.choices( - ( - f"{prefix}MODEL ({separator} {header}{separator});{separator}{body}", - f"MODEL ({header});", - ), - weights=(19, 1), - )[0] - return text.encode("utf-8") - - -LAYOUT_ROOTS: tuple[str, ...] = ("models",) * 8 + ( - "tests/unit", "tests/scenarios", "functions/sql", "sources", "macros", "enums", "constants", - "audits", "schemas", "hooks", "_macros", "_sqlbuild", "seeds", -) # fmt: skip -LAYOUT_SEGMENTS: tuple[str, ...] = ("marts", "core") * 6 + ( - "_sqlbuild", "_sqlbuild", "_sqlbuild", "macros", "enums", "constants", "_macros", "_enums", - "_constants", "audits", "_audits", "generic", "singular", "schemas", "_schemas", "hooks", - "_hooks", "sql", "python", ".cache", -) # fmt: skip -LAYOUT_TEMPLATES: tuple[str, ...] = ( - "{root}/{owner}/_sqlbuild/{declaration}", "{root}/{owner}/{local}", "{declaration}", - "{declaration}/{owner}", "{root}/{owner}/_sqlbuild/{named}", "{named}", "{root}/{owner}", -) # fmt: skip -LAYOUT_DECLARATIONS: tuple[str, ...] = ( - "macros", "enums", "constants", "_macros", "_enums", "_constants", -) # fmt: skip -LAYOUT_NAMED_ROLES: tuple[str, ...] = ( - "audits/generic", "audits/singular", "_audits/generic", "schemas", "_schemas", "hooks/sql", - "hooks/python", "_hooks/sql", -) # fmt: skip -LAYOUT_FILE_NAMES: tuple[str, ...] = ("a.py", "b.sql", "c.sql", "__init__.py", "notes.txt", ".keep") -LAYOUT_CONTENTS: dict[str, str] = { - "enums": "ENUM (name e{n}, members [A]);\n", - "_enums": "ENUM (name e{n}, members [A]);\n", - "constants": "CONSTANT (name c{n}, value 1);\n", - "_constants": "CONSTANT (name c{n}, value 1);\n", -} -_FACT_KINDS: dict[type[object], str] = { - DiscoveredMacroFile: "macro", - DiscoveredEnumFile: "enum", - DiscoveredConstantFile: "constant", -} - - -def generated_layout_files(*, rng: random.Random) -> tuple[tuple[str, bytes], ...]: - """Return files spread over canonical, global and grouped declaration roots.""" +def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch) -> object: + """Return the rendered discovery stage under `engine` with its failure type and message.""" - files: dict[str, bytes] = {} - for index in range(rng.randint(1, 6)): - directory: str = rng.choices( - ( - rng.choice(LAYOUT_TEMPLATES).format( - root=rng.choice(LAYOUT_ROOTS[:12]), - owner=rng.choice(("marts", "core", "marts/core")), - declaration=rng.choice(LAYOUT_DECLARATIONS), - local=rng.choice(LAYOUT_DECLARATIONS[3:]), - named=rng.choice(LAYOUT_NAMED_ROLES), - ), - "/".join( - (rng.choice(LAYOUT_ROOTS), *rng.choices(LAYOUT_SEGMENTS, k=rng.randint(0, 3))) - ), - ), - weights=(6, 1), - )[0] - parts: list[str] = [*directory.split("/"), rng.choice(LAYOUT_FILE_NAMES)] - template: str = next( - filter(None, (LAYOUT_CONTENTS.get(part) for part in reversed(parts))), "" - ) - files["/".join(parts)] = template.format(n=index).encode("utf-8") - return tuple(files.items()) + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) + capture: FailureCapture = FailureCapture() + rendered: list[object] = [None] + with capture: + rendered[0] = render_stage_capture(discover_project_inputs(project_dir=project_dir)) + return (rendered[0], type(capture.failure).__name__, str(capture.failure)) -def python_layout_outcome(*, project_dir: Path) -> tuple[object, object]: - """Return Python's declaration file facts and named roots, `None` where its scan raises.""" +def random_undecodable_bytes(*, rng: random.Random) -> bytes: + """Return text bytes with UTF-8 sequences cut, corrupted or replaced by invalid bytes.""" - return ( - _outcome_or_none(lambda: _python_fact_rows(project_dir=project_dir)), - _outcome_or_none(lambda: _named_root_rows(project_dir=project_dir, seed=lambda: None)), - ) + text: bytes = random_text(rng=rng, max_length=12).encode("utf-8") + cut: int = rng.randint(0, len(text)) + noise: bytes = bytes(rng.choice(UNDECODABLE_BYTES) for _ in range(rng.randint(0, 3))) + return rng.choice((text[:cut] + noise + text[cut:], text[:cut], noise + text)) -def native_layout_outcome(*, project_dir: Path) -> tuple[object, object]: - """Return the native facts and the named roots its groups give, `None` where it fails.""" +def native_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: + """Return the type and text of the read error native reading reports for each file, if any.""" - layout: tuple[list[NativeDeclarationFact] | None, list[tuple[str, str]] | None] | None = ( - _native.discover_declaration_layout(_native.NativeProjectTree(str(project_dir))) - ) - facts, groups = layout or (None, None) - return ( - _mapped(facts, lambda rows: [_native_fact_row(fact) for fact in rows]), - _mapped( - groups, - lambda rows: _named_root_rows( - project_dir=project_dir, - seed=lambda: remember_declaration_groups(project_dir=project_dir, groups=rows), - ), + payloads: list[tuple[object, ...]] = cast( + list[tuple[object, ...]], + _native.load_yaml_files( + {"project_dir": str(project_dir), "display_prefix": "", "kind": "source"}, + relative_paths, + _native.NativeProjectTree(str(project_dir)), ), ) + errors: list[Exception | None] = [ + native_payload_error(payload=payload, file_path=project_dir / path) + for payload, path in zip(payloads, relative_paths, strict=True) + ] + return [ + {True: (type(error).__name__, str(error)), False: ("NoneType", "None")}[ + isinstance(error, UnicodeError | OSError) + ] + for error in errors + ] -def _mapped[T](value: T | None, transform: Callable[[T], object]) -> object: - return {True: lambda: None, False: lambda: transform(cast(T, value))}[value is None]() - - -def _outcome_or_none(compute: Callable[[], object]) -> object: - outcome: list[object] = [None] - with suppress(DeclarationParseError): - outcome[0] = compute() - return outcome[0] - - -def _python_fact_rows(*, project_dir: Path) -> list[tuple[str, ...]]: - with DirectorySnapshot.scope(project_dir=project_dir): - files: tuple[DiscoveredMacroFile | DiscoveredEnumFile | DiscoveredConstantFile, ...] = ( - *discover_macro_files(project_dir=project_dir), - *discover_enum_files(project_dir=project_dir), - *discover_constant_files(project_dir=project_dir), - ) - return sorted( - ( - file.relative_path.as_posix(), - _FACT_KINDS[type(file)], - file.scope_kind.value, - str(file.ownership_root), - str(file.owning_path), - str(file.declaration_root), - ) - for file in files - ) +def python_read_outcomes(*, project_dir: Path, relative_paths: list[str]) -> list[object]: + """Return the type and text of the error `Path.read_text` raises for each file, if any.""" + outcomes: list[object] = [] + for relative_path in relative_paths: + capture: FailureCapture = FailureCapture() + with capture: + _ = (project_dir / relative_path).read_text(encoding="utf-8") + outcomes.append((type(capture.failure).__name__, str(capture.failure))) + return outcomes -def _native_fact_row(fact: NativeDeclarationFact) -> tuple[str, ...]: - relative_path, kind, scope_kind, ownership_root, owning_path, declaration_root = fact - return (relative_path, kind, scope_kind, ownership_root, str(owning_path), declaration_root) +def compile_failure(*, project_dir: Path) -> tuple[str, str]: + """Return the type and text of the error project discovery raises.""" -def _named_root_rows(*, project_dir: Path, seed: Callable[[], None]) -> list[tuple[str, ...]]: - with DirectorySnapshot.scope(project_dir=project_dir): - seed() - return [ - ( - root.relative_directory.as_posix(), - root.kind.value, - root.scope_kind.value, - str(root.ownership_root), - str(root.owning_path), - ) - for root in named_declaration_roots( - project_dir=project_dir, kinds=NAMED_DECLARATION_KINDS - ) - ] + capture: FailureCapture = FailureCapture() + with capture: + _ = discover_project_inputs(project_dir=project_dir) + return type(capture.failure).__name__, str(capture.failure) -def stage_outcome(*, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch) -> object: - """Return the rendered discovery stage under `engine` with its failure type and message.""" +def discovery_failure_with_help(*, project_dir: Path) -> tuple[str, str, bool]: + """Return the type and text of the error project discovery raises, and whether it has help.""" - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) capture: FailureCapture = FailureCapture() - rendered: list[object] = [None] with capture: - rendered[0] = render_stage_capture(discover_project_inputs(project_dir=project_dir)) - return (rendered[0], type(capture.failure).__name__, str(capture.failure)) + _ = discover_project_inputs(project_dir=project_dir) + return ( + type(capture.failure).__name__, + str(capture.failure), + getattr(capture.failure, "help", None) is not None, + ) -class RecordingFactCache: - """A fact cache that records the fact keys discovery asks for and caches nothing.""" +def tolerant_scope_fault_outcome(*, project_dir: Path) -> tuple[object, ...]: + """Return tolerant scope discovery's model and source paths and its per-file faults.""" - enabled: bool = True + discovery: TolerantScopeDiscovery = build_tolerant_scope_discovery(project_dir=project_dir) + return ( + *_inputs_paths(discovery.discovered_inputs), + tuple( + (str(fault.path), fault.message) + for fault in (*discovery.resource_faults, *discovery.relationship_faults) + ), + ) - def __init__(self) -> None: - self.keys: list[tuple[str, ...]] = [] - def key(self, *parts: str) -> str: - self.keys.append(parts) - return str(len(self.keys)) +def description_inputs_outcome(*, project_dir: Path) -> tuple[object, ...]: + """Return the model and source paths model description resolution reads, without faults.""" - def read_many(self, entries: object) -> dict[str, object]: - return {} + return (*_inputs_paths(discover_model_description_inputs(project_dir=project_dir)), ()) - def stage(self, *, key: str, slot: str, value: object) -> None: - return None +def selected_contract_outcome(*, project_dir: Path) -> tuple[object, ...]: + """Return the model and source paths selected-contract discovery reads for `orders`.""" -def fact_cache_keys(*, project_dir: Path, native: bool) -> list[tuple[str, ...]]: - """Return the fact keys source and test discovery request under the chosen path.""" + inputs: DiscoveredProjectInputs = SelectedContractInputDiscoverer.discover( + project_dir=project_dir, + selected_test_paths=frozenset(), + referenced_model_names=frozenset({"orders"}), + ) + return (*_inputs_paths(inputs), ()) - cache: RecordingFactCache = RecordingFactCache() - fact_cache: FactCacheStore = cast(FactCacheStore, cache) - discover: Callable[[], object] = { - True: lambda: ( - discover_native_source_files(project_dir=project_dir, fact_cache=fact_cache), - discover_native_test_files(project_dir=project_dir, fact_cache=fact_cache), - ), - False: lambda: ( - discover_source_files(project_dir=project_dir, fact_cache=fact_cache), - discover_test_files(project_dir=project_dir, fact_cache=fact_cache), - ), - }[native] - with DirectorySnapshot.scope(project_dir=project_dir): - _ = discover() - return cache.keys +def _inputs_paths(inputs: DiscoveredProjectInputs) -> tuple[object, ...]: + return ( + tuple(model.relative_path.as_posix() for model in inputs.model_files), + tuple(source.relative_path.as_posix() for source in inputs.source_files), + ) -class CallCounter: - """Count calls and delegate to the wrapped function.""" - def __init__(self, function: Callable[..., object]) -> None: - self.function: Callable[..., object] = function - self.calls: int = 0 +def declared_enums_outcome(*, project_dir: Path) -> tuple[object, ...]: + """Return each discovered enum file with the enum names it declares, and the model paths.""" - def __call__(self, *args: object) -> object: - self.calls += 1 - return self.function(*args) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + return ( + tuple(_declared_enum_names(enum_file) for enum_file in inputs.enum_files), + tuple(model.relative_path.as_posix() for model in inputs.model_files), + ) + + +def _declared_enum_names(enum_file: DiscoveredEnumFile) -> tuple[str, tuple[str, ...]]: + return ( + enum_file.relative_path.as_posix(), + tuple(declaration.name for declaration in enum_file.declarations), + ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py index a3b9b3b16d..27e9443978 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_discovery.py @@ -2,23 +2,17 @@ from __future__ import annotations -import random from pathlib import Path import pytest from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( EngineSwitchParityTestCase, - GeneratedLayoutParityTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - generated_layout_files, - native_layout_outcome, - python_layout_outcome, stage_outcome, write_project, ) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches _AUDIT: bytes = b'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' _SCHEMA: bytes = ( @@ -76,42 +70,3 @@ def test_given_declaration_layout_when_discovering_through_the_engine_switch_the native: object = stage_outcome(project_dir=tmp_path, engine="native", monkeypatch=monkeypatch) assert (native == python) is test_case.expected_identical, test_case.description - - -@pytest.mark.parametrize( - "test_case", - [ - GeneratedLayoutParityTestCase( - description="seeded layouts over every declaration role", - seed=20261007, - count=400, - expected_minimum_valid=60, - expected_minimum_invalid=60, - ), - ], - ids=lambda case: case.description, -) -def test_given_generated_layouts_when_validating_natively_then_facts_and_failures_match_python( - test_case: GeneratedLayoutParityTestCase, tmp_path: Path -) -> None: - rng: random.Random = random.Random(test_case.seed) - project_dirs: list[Path] = [tmp_path / f"project_{index}" for index in range(test_case.count)] - for project_dir in project_dirs: - write_project(project_dir=project_dir, files=generated_layout_files(rng=rng)) - expected: list[tuple[object, object]] = [ - python_layout_outcome(project_dir=path) for path in project_dirs - ] - - actual: list[tuple[object, object]] = [ - native_layout_outcome(project_dir=path) for path in project_dirs - ] - - valid: int = sum(None not in outcome for outcome in expected) - inputs: list[object] = [path.name for path in project_dirs] - expected_outcomes: list[object] = [*expected] - actual_outcomes: list[object] = [*actual] - assert ( - mismatches(inputs=inputs, expected=expected_outcomes, actual=actual_outcomes), - valid >= test_case.expected_minimum_valid, - test_case.count - valid >= test_case.expected_minimum_invalid, - ) == (list(test_case.expected_mismatches), True, True), test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index da3da2ab9b..400211148f 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -3,131 +3,36 @@ from __future__ import annotations import os -import random import sys import unicodedata from pathlib import Path import pytest -import sqlbuild._native as _native +from sqlbuild import _native +from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.discovery._helpers.native.model_files import ( discover_native_model_files, ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( EngineSwitchParityTestCase, - GeneratedModelParityTestCase, - ModelDiscoveryParityTestCase, - NativeDeferralTestCase, + NativeRuntimeTestCase, SharedSnapshotTestCase, + UnsupportedPythonCommandTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - CallCounter, - generated_model_bytes, - model_discovery_outcome, + FailureCapture, write_project, ) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches _VALID_MODEL: bytes = b"MODEL (materialized table);\nSELECT order_id, total AS amount FROM orders" -@pytest.mark.parametrize( - "test_case", - [ - ModelDiscoveryParityTestCase( - description="sorted part by part, scoped declaration trees skipped", - files=( - ("models/a-c.sql", _VALID_MODEL), - ("models/a/b.sql", _VALID_MODEL), - ("models/.hidden.sql", _VALID_MODEL), - ("models/core/_sqlbuild/schemas/x.sql", b"not a model"), - ("models/core/macros/helpers.sql", b"not a model"), - ("models/notes.txt", b"ignored"), - ), - ), - ModelDiscoveryParityTestCase( - description="CRLF, CR and non-ASCII locations", - files=( - ( - "models/orders.sql", - "MODEL (\r\n description 'é',\r columns (id (type INT)),\r\n);\r\n" - 'SELECT id, "Ünit" AS ünit FROM t'.encode(), - ), - ), - ), - ModelDiscoveryParityTestCase( - description="byte order mark before the header", - files=(("models/orders.sql", b"\xef\xbb\xbf" + _VALID_MODEL),), - ), - ModelDiscoveryParityTestCase( - description="invalid UTF-8 raises Python's decoding error", - files=(("models/a.sql", _VALID_MODEL), ("models/b.sql", b"MODEL ();\nSELECT '\xff'")), - ), - ModelDiscoveryParityTestCase( - description="a directory named like a model is read and fails", - files=(("models/a.sql", _VALID_MODEL),), - directories=("models/b.sql",), - ), - ModelDiscoveryParityTestCase( - description="the first failing file in order wins", - files=( - ("models/a.sql", _VALID_MODEL), - ("models/b.sql", b"MODEL (tagz [x]);\nSELECT 1"), - ("models/c.sql", b"SELECT 1"), - ), - ), - ModelDiscoveryParityTestCase( - description="model-local enum declarations fail after the header checks", - files=(("models/a.sql", b"MODEL (enums [a]);\nSELECT 1"),), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_files_when_discovering_with_each_engine_then_outcomes_match( - test_case: ModelDiscoveryParityTestCase, tmp_path: Path -) -> None: - write_project(project_dir=tmp_path, files=test_case.files) - for directory in test_case.directories: - (tmp_path / directory).mkdir(parents=True) - - python: object = model_discovery_outcome(project_dir=tmp_path, native=False) - native: object = model_discovery_outcome(project_dir=tmp_path, native=True) - - assert (native == python) is test_case.expected_identical - - -@pytest.mark.parametrize( - "test_case", - [ - GeneratedModelParityTestCase( - description="mixed headers and projections", seed=61, case_count=400 - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_model_files_when_discovering_with_each_engine_then_outcomes_match( - test_case: GeneratedModelParityTestCase, tmp_path: Path -) -> None: - rng: random.Random = random.Random(test_case.seed) - contents: list[bytes] = [generated_model_bytes(rng=rng) for _ in range(test_case.case_count)] - expected: list[object] = [] - actual: list[object] = [] - for index, data in enumerate(contents): - project_dir: Path = tmp_path / f"case_{index}" - write_project(project_dir=project_dir, files=(("models/orders.sql", data),)) - expected.append(model_discovery_outcome(project_dir=project_dir, native=False)) - actual.append(model_discovery_outcome(project_dir=project_dir, native=True)) - - assert mismatches(inputs=list(contents), expected=expected, actual=actual) == list( - test_case.expected_mismatches - ) - - @pytest.mark.parametrize( "test_case", [ @@ -160,53 +65,69 @@ def test_given_project_when_discovering_through_the_engine_switch_then_inputs_ma @pytest.mark.parametrize( "test_case", [ - NativeDeferralTestCase( - description="matching Unicode data and a UTF-8 root run natively", + NativeRuntimeTestCase( + description="the running Python and a UTF-8 root run natively", project_name="orders", - expected_native_calls=1, + expected_models=1, ), - NativeDeferralTestCase( - description="different Unicode data defers to Python", - project_name="orders", - unidata_version="0.0.0", - expected_native_calls=0, - ), - NativeDeferralTestCase( + NativeRuntimeTestCase( description="another supported Python and its Unicode data run natively", project_name="orders", unidata_version="16.0.0", python_version=(3, 14), - expected_native_calls=1, + expected_models=1, ), - NativeDeferralTestCase( - description="an unreleased Python defers to Python", + NativeRuntimeTestCase( + description="unknown Unicode data fails naming the supported versions", + project_name="orders", + unidata_version="0.0.0", + expected_models=0, + expected_error="UnsupportedPythonError", + expected_message="This SQLBuild release supports Python 3.12, 3.13 and 3.14, not Python", + ), + NativeRuntimeTestCase( + description="an unreleased Python fails naming its version", project_name="orders", unidata_version="16.0.0", python_version=(3, 15), - expected_native_calls=0, + expected_models=0, + expected_error="UnsupportedPythonError", + expected_message="not Python 3.15 (Unicode 16.0.0); run sqb with a supported Python", ), - NativeDeferralTestCase( - description="a non-UTF-8 project root defers to Python", + NativeRuntimeTestCase( + description="a non-UTF-8 project root fails showing the path lossily", project_name=os.fsdecode(b"orders\xff"), - expected_native_calls=0, + expected_models=0, + expected_error="ProjectPathError", + expected_message="orders\\xff is not valid UTF-8; move the project to a directory", ), ], ids=lambda case: case.description, ) -def test_given_runtime_and_root_when_discovering_natively_then_python_runs_where_native_cannot_match( - test_case: NativeDeferralTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_given_runtime_and_root_when_discovering_natively_then_supported_runs_and_others_fail( + test_case: NativeRuntimeTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: project_dir: Path = tmp_path / test_case.project_name write_project(project_dir=project_dir, files=(("models/orders.sql", _VALID_MODEL),)) - python: object = model_discovery_outcome(project_dir=project_dir, native=False) - counter: CallCounter = CallCounter(_native.discover_model_files) - monkeypatch.setattr(_native, "discover_model_files", counter) monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) monkeypatch.setattr(sys, "version_info", (*test_case.python_version, 0, "final", 0)) + capture: FailureCapture = FailureCapture() + models: list[DiscoveredSqlModelFile] = [] + + with capture: + models.extend( + discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + ) + ) - native: object = model_discovery_outcome(project_dir=project_dir, native=True) - - assert (native, counter.calls) == (python, test_case.expected_native_calls) + assert ( + len(models), + type(capture.failure).__name__, + test_case.expected_message in str(capture.failure), + ) == (test_case.expected_models, test_case.expected_error, True), test_case.description @pytest.mark.parametrize( @@ -238,3 +159,41 @@ def test_given_native_model_walk_when_globbing_models_later_in_the_pass_then_the assert tuple(path.relative_to(tmp_path).as_posix() for path in matches) == ( test_case.expected_matches ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnsupportedPythonCommandTestCase( + description="compile reports the coded error without a traceback", + command=("compile",), + expected_exit_code=1, + expected_error="error[D017]: This SQLBuild release supports Python 3.12, 3.13 and 3.14", + ), + UnsupportedPythonCommandTestCase( + description="tolerant scope discovery still refuses the Python", + command=("scope", "models/orders.sql"), + expected_exit_code=1, + expected_error="error[D017]: This SQLBuild release supports Python 3.12, 3.13 and 3.14", + ), + ], + ids=lambda case: case.description, +) +def test_given_unsupported_python_when_running_command_then_coded_error_is_printed( + test_case: UnsupportedPythonCommandTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project(project_dir=tmp_path, files=(("models/orders.sql", _VALID_MODEL),)) + monkeypatch.setattr(_native, "native_text_supported", lambda *_arguments: False) + _ = capsys.readouterr() + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command]) + + error: str = capsys.readouterr().err + assert (exit_code, test_case.expected_error in error, "Traceback" in error) == ( + test_case.expected_exit_code, + True, + False, + ), error diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py new file mode 100644 index 0000000000..af15c466c9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_read_errors.py @@ -0,0 +1,166 @@ +"""Native discovery reports unreadable authored files with Python's own read errors.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery._helpers.native.payloads import native_read_error +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + NativeOsErrorTestCase, + ReadErrorOracleTestCase, + UnreadableFileTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + compile_failure, + native_read_outcomes, + python_read_outcomes, + random_undecodable_bytes, + write_project, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_STAGING: bytes = b'MODEL (description "Orders");\nSELECT 1 AS order_id' + + +@pytest.mark.parametrize( + "test_case", + [ + ReadErrorOracleTestCase( + description="seeded cut, corrupted and invalid UTF-8 sequences", + seed=97, + case_count=3000, + expected_minimum_failures=1500, + ) + ], + ids=lambda case: case.description, +) +def test_given_undecodable_files_when_reading_natively_then_errors_equal_read_text_errors( + test_case: ReadErrorOracleTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: tuple[tuple[str, bytes], ...] = tuple( + (f"sources/file_{index}.yml", random_undecodable_bytes(rng=rng)) + for index in range(test_case.case_count) + ) + write_project(project_dir=tmp_path, files=files) + relative_paths: list[str] = [path for path, _data in files] + python: list[object] = python_read_outcomes(project_dir=tmp_path, relative_paths=relative_paths) + + native: list[object] = native_read_outcomes(project_dir=tmp_path, relative_paths=relative_paths) + + failures: int = sum(outcome != ("NoneType", "None") for outcome in python) + assert ( + mismatches(inputs=list(relative_paths), expected=python, actual=native), + failures >= test_case.expected_minimum_failures, + ) == ([], True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + UnreadableFileTestCase( + description="an undecodable model", + relative_path="models/orders.sql", + data=_STAGING + b"\xff", + expected_error="UnicodeDecodeError", + expected_message=( + "'utf-8' codec can't decode byte 0xff in position 50: invalid start byte" + ), + ), + UnreadableFileTestCase( + description="a SQL test cut inside a sequence", + relative_path="tests/unit/orders.sql", + data=b'TEST (name "keeps");\nSELECT \xe2\x82', + expected_error="UnicodeDecodeError", + expected_message=( + "'utf-8' codec can't decode bytes in position 28-29: unexpected end of data" + ), + ), + UnreadableFileTestCase( + description="a scenario with an invalid continuation byte", + relative_path="tests/scenarios/orders.sql", + data=b'SCENARIO (description "x");\nSELECT \xe2\x82x', + expected_error="UnicodeDecodeError", + expected_message=( + "'utf-8' codec can't decode bytes in position 35-36: invalid continuation byte" + ), + ), + UnreadableFileTestCase( + description="an undecodable source file", + relative_path="sources/raw.yml", + data=b"sources: []\n# \xc0\n", + expected_error="UnicodeDecodeError", + expected_message=( + "'utf-8' codec can't decode byte 0xc0 in position 14: invalid start byte" + ), + ), + UnreadableFileTestCase( + description="a directory named like a schema file", + relative_path="models/marts/schema.yml/notes.txt", + data=b"", + expected_error="IsADirectoryError", + expected_message="[Errno 21] Is a directory: '{project}/models/marts/schema.yml'", + ), + ], + ids=lambda case: case.description, +) +def test_given_unreadable_file_when_discovering_then_python_read_error_is_raised( + test_case: UnreadableFileTestCase, tmp_path: Path +) -> None: + write_project( + project_dir=tmp_path, + files=(("models/staging.sql", _STAGING), (test_case.relative_path, test_case.data)), + ) + + failure: tuple[str, str] = compile_failure(project_dir=tmp_path) + + assert failure == ( + test_case.expected_error, + test_case.expected_message.format(project=tmp_path), + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + NativeOsErrorTestCase( + description="a refused read is Python's errno error naming the file", + payload=("read", "os", 13, "Permission denied (os error 13)", None), + expected_type="PermissionError", + expected_fields=(13, "Permission denied", "orders.sql"), + ), + NativeOsErrorTestCase( + description="a Windows listing error carries its Win32 code for Python to map", + payload=("read", "os", None, "Access is denied", 5), + expected_type="OSError", + expected_fields=(None, "Access is denied", "orders.sql"), + ), + NativeOsErrorTestCase( + description="an error without a code keeps the operating system message", + payload=("read", "os", None, "stream did not contain valid UTF-8", None), + expected_type="OSError", + expected_fields=(None, None, None), + ), + ], + ids=lambda case: case.description, +) +def test_given_os_read_payload_when_building_error_then_python_error_matches_platform( + test_case: NativeOsErrorTestCase, +) -> None: + error: Exception = native_read_error(payload=test_case.payload, file_path=Path("orders.sql")) + + assert ( + type(error).__name__, + ( + getattr(error, "errno", None), + getattr(error, "strerror", None), + getattr(error, "filename", None), + ), + ) == (test_case.expected_type, test_case.expected_fields), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py index d4c478137f..957191f4c1 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_sql_test_discovery.py @@ -2,73 +2,22 @@ from __future__ import annotations -import random from pathlib import Path import pytest from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( EngineSwitchParityTestCase, - GeneratedSqlTestParityTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - generated_scenario_bytes, - generated_test_bytes, - sql_test_discovery_outcome, stage_outcome, write_project, ) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches _TEST: bytes = b'TEST (name "keeps_orders");\n\nSELECT 1\n' _SCENARIO: bytes = b'SCENARIO (description "Orders world");\n\nSELECT 1\n' -@pytest.mark.parametrize( - "test_case", - [ - GeneratedSqlTestParityTestCase( - description="seeded test blocks and scenario headers", - seed=71, - case_count=1000, - expected_minimum_parsed=80, - expected_minimum_failed=150, - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_test_files_when_discovering_with_each_engine_then_outcomes_match( - test_case: GeneratedSqlTestParityTestCase, tmp_path: Path -) -> None: - rng: random.Random = random.Random(test_case.seed) - files: list[tuple[tuple[str, bytes], ...]] = [ - ( - ("tests/unit/orders.sql", generated_test_bytes(rng=rng)), - ("tests/scenarios/orders.sql", generated_scenario_bytes(rng=rng)), - ) - for _ in range(test_case.case_count) - ] - project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] - for project_dir, project_files in zip(project_dirs, files, strict=True): - write_project(project_dir=project_dir, files=project_files) - expected: list[object] = [ - sql_test_discovery_outcome(project_dir=project_dir, native=False) - for project_dir in project_dirs - ] - - actual: list[object] = [ - sql_test_discovery_outcome(project_dir=project_dir, native=True) - for project_dir in project_dirs - ] - - parsed: int = sum(isinstance(outcome, str) for outcome in expected) - assert ( - mismatches(inputs=list(files), expected=expected, actual=actual), - parsed >= test_case.expected_minimum_parsed, - len(expected) - parsed >= test_case.expected_minimum_failed, - ) == (list(test_case.expected_mismatches), True, True), test_case.description - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py new file mode 100644 index 0000000000..e84753c024 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_undecodable_paths.py @@ -0,0 +1,163 @@ +"""Callers that tolerate bad files fault only the files whose read paths are not UTF-8.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + CollidingNamesTestCase, + UndecodablePathToleranceTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + declared_enums_outcome, + description_inputs_outcome, + selected_contract_outcome, + tolerant_scope_fault_outcome, + write_project, +) + +_MODEL: bytes = b"MODEL ();\nSELECT 1 AS id" +_TEST: bytes = b'TEST (name "keeps_orders");\nSELECT 1\n' +_SOURCE: bytes = b"sources:\n - name: raw_orders\n" +_FILES: tuple[tuple[str, bytes], ...] = ( + ("models/orders.sql", _MODEL), + (os.fsdecode(b"models/caf\xe9.sql"), _MODEL), + (os.fsdecode(b"models/notes\xe9.txt"), b"x"), + ("tests/unit/orders.sql", _TEST), + (os.fsdecode(b"tests/unit/old\xe9/orders.sql"), _TEST), + ("sources/raw.yml", _SOURCE), +) +_BAD_SOURCE: tuple[str, bytes] = (os.fsdecode(b"sources/caf\xe9.yml"), _SOURCE) +_RENAME: str = " is not valid UTF-8; rename it so SQLBuild can read it" +_ORDER_STATUS: bytes = b"ENUM (name order_status, members [PLACED]);\n" +_TIER: bytes = b"ENUM (name tier, members [GOLD]);\n" +_READERS: dict[str, Callable[..., tuple[object, ...]]] = { + "scope": tolerant_scope_fault_outcome, + "descriptions": description_inputs_outcome, + "selected contracts": selected_contract_outcome, +} + + +@pytest.mark.parametrize( + "test_case", + [ + UndecodablePathToleranceTestCase( + description="scope discovery faults each unreadable path and keeps the rest", + reader="scope", + files=(*_FILES, _BAD_SOURCE), + expected_models=("models/orders.sql",), + expected_sources=("sources/raw.yml",), + expected_faults=( + (_FILES[1][0], f"Project path ./models/caf\\xe9.sql{_RENAME}"), + (_BAD_SOURCE[0], f"Project path ./sources/caf\\xe9.yml{_RENAME}"), + (_FILES[4][0], f"Project path ./tests/unit/old\\xe9/orders.sql{_RENAME}"), + ), + ), + UndecodablePathToleranceTestCase( + description="a valid name is not blamed for a raw sibling with the same lossy name", + reader="scope", + files=( + ("models/orders.sql", _MODEL), + ("models/st\ufffd.sql", b"MODEL ();\nSELECT 2 AS id"), + (os.fsdecode(b"models/st\xe8.sql"), b"garbage"), + ), + expected_models=("models/orders.sql", "models/st\ufffd.sql"), + expected_sources=(), + expected_faults=( + ( + os.fsdecode(b"models/st\xe8.sql"), + f"Project path ./models/st\\xe8.sql{_RENAME}", + ), + ), + ), + UndecodablePathToleranceTestCase( + description="model descriptions skip an unreadable model path", + reader="descriptions", + files=(*_FILES, _BAD_SOURCE), + expected_models=("models/orders.sql",), + expected_sources=(), + ), + UndecodablePathToleranceTestCase( + description="selected contracts never read an unselected model path", + reader="selected contracts", + files=_FILES, + expected_models=("models/orders.sql",), + expected_sources=("sources/raw.yml",), + ), + ], + ids=lambda case: case.description, +) +def test_given_undecodable_paths_when_discovering_tolerantly_then_each_read_path_faults_alone( + test_case: UndecodablePathToleranceTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + outcome: tuple[object, ...] = _READERS[test_case.reader](project_dir=tmp_path) + + assert outcome == ( + test_case.expected_models, + test_case.expected_sources, + test_case.expected_faults, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + CollidingNamesTestCase( + description="an enum file whose name is not UTF-8 is read by its real name", + files=((os.fsdecode(b"enums/st\xe9.sql"), _ORDER_STATUS),), + expected_enums=((os.fsdecode(b"enums/st\xe9.sql"), ("order_status",)),), + ), + CollidingNamesTestCase( + description="enum files sharing a lossy name each keep their own declarations", + files=( + (os.fsdecode(b"enums/st\xe9.sql"), _ORDER_STATUS), + (os.fsdecode(b"enums/st\xe8.sql"), _TIER), + ), + expected_enums=( + (os.fsdecode(b"enums/st\xe8.sql"), ("tier",)), + (os.fsdecode(b"enums/st\xe9.sql"), ("order_status",)), + ), + ), + CollidingNamesTestCase( + description="a valid replacement-character name is not replaced by a raw sibling", + files=( + ("enums/st\ufffd.sql", _ORDER_STATUS), + (os.fsdecode(b"enums/st\xe8.sql"), _TIER), + ), + expected_enums=( + (os.fsdecode(b"enums/st\xe8.sql"), ("tier",)), + ("enums/st\ufffd.sql", ("order_status",)), + ), + ), + CollidingNamesTestCase( + description="directories sharing a lossy name each keep their own files", + files=( + (os.fsdecode(b"models/a\xe9/_enums/x.sql"), _ORDER_STATUS), + (os.fsdecode(b"models/a\xe8/_enums/y.sql"), _TIER), + ), + expected_enums=( + (os.fsdecode(b"models/a\xe8/_enums/y.sql"), ("tier",)), + (os.fsdecode(b"models/a\xe9/_enums/x.sql"), ("order_status",)), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_names_sharing_a_lossy_spelling_when_discovering_then_each_file_keeps_its_identity( + test_case: CollidingNamesTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=(("models/orders.sql", _MODEL), *test_case.files)) + + outcome: tuple[object, ...] = declared_enums_outcome(project_dir=tmp_path) + + assert outcome == (test_case.expected_enums, test_case.expected_models), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_and_path_failures.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_and_path_failures.py new file mode 100644 index 0000000000..edb6db645f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_and_path_failures.py @@ -0,0 +1,165 @@ +"""Discovery reads plain YAML and UTF-8 paths, and rejects anything else with one clear error.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + AuthoredFileFailureTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + discovery_failure_with_help, + write_project, +) + +_SOURCE_HEAD: bytes = b"sources:\n - name: raw_orders\n" +_MODEL: bytes = b"MODEL ();\nSELECT 1 AS id" +_HELP: bool = True + + +@pytest.mark.parametrize( + "test_case", + [ + AuthoredFileFailureTestCase( + description="block, flow, quoted, folded and literal values with anchors and merges", + files=( + ( + "sources/raw.yml", + b"---\n# raw\n" + _SOURCE_HEAD + b" description: >\n Raw\n" + b" orders.\n meta: &m {owner: 'data', tags: [a, \"b\"]}\n" + b" - name: raw_items\n description: |\n Items.\n" + b" meta:\n <<: *m\n extra: ~\n", + ), + ), + ), + AuthoredFileFailureTestCase( + description="invalid YAML names the file and position", + files=(("sources/raw.yml", _SOURCE_HEAD + b" description: [unclosed\n"),), + expected_error="SourceParseError", + expected_message=( + "{project}/sources/raw.yml contains invalid YAML at line 4, column 1: while " + "parsing a flow sequence, expected ',' or ']'" + ), + ), + AuthoredFileFailureTestCase( + description="a binary value is unsupported at its node", + files=(("sources/raw.yml", _SOURCE_HEAD + b" meta:\n blob: !!binary aGk=\n"),), + expected_error="SourceParseError", + expected_message=( + "{project}/sources/raw.yml uses YAML that SQLBuild does not support at line 4, " + "column 22: values tagged !!binary" + ), + expected_help=_HELP, + ), + AuthoredFileFailureTestCase( + description="a tab is unsupported at its character", + files=(("sources/raw.yml", _SOURCE_HEAD + b' description: "Raw\torders"\n'),), + expected_error="SourceParseError", + expected_message=( + "{project}/sources/raw.yml uses YAML that SQLBuild does not support at line 3, " + "column 22: tab characters" + ), + expected_help=_HELP, + ), + AuthoredFileFailureTestCase( + description="a directive is unsupported at its line", + files=(("sources/raw.yml", b"%YAML 1.1\n---\nsources: []\n"),), + expected_error="SourceParseError", + expected_message=( + "{project}/sources/raw.yml uses YAML that SQLBuild does not support at line 1, " + "column 1: %YAML and %TAG directives" + ), + expected_help=_HELP, + ), + AuthoredFileFailureTestCase( + description="a recursive alias is unsupported at the alias", + files=(("sources/raw.yml", b"sources: []\nmeta: &a [*a]\n"),), + expected_error="SourceParseError", + expected_message=( + "{project}/sources/raw.yml uses YAML that SQLBuild does not support at line 2, " + "column 11: recursive aliases" + ), + expected_help=_HELP, + ), + AuthoredFileFailureTestCase( + description="an invalid schema file fails as a schema file", + files=(("models/marts/schema.yml", b"models: [\n"),), + expected_error="SchemaParseError", + expected_message=( + "{project}/models/marts/schema.yml contains invalid YAML at line 2, column 1: " + "while parsing a node, did not find expected node content" + ), + ), + AuthoredFileFailureTestCase( + description="names discovery never reads may be any bytes", + files=( + ("models/orders.sql", _MODEL), + (os.fsdecode(b"models/notes\xe9.txt"), b"x"), + (os.fsdecode(b"models/old\xe9/readme.md"), b"x"), + (os.fsdecode(b"macros/x\xe9.txt"), b"x"), + (os.fsdecode(b"tests/unit/x\xe9.txt"), b"x"), + (os.fsdecode(b"tests/scenarios/x\xe9.md"), b"x"), + (os.fsdecode(b"sources/x\xe9.txt"), b"x"), + (os.fsdecode(b"seeds/x\xe9.txt"), b"x"), + (os.fsdecode(b"data\xe9.csv"), b"x"), + (os.fsdecode(b"target/x\xe9"), b"x"), + ), + ), + AuthoredFileFailureTestCase( + description="a model below a directory that is not UTF-8 is shown lossily", + files=((os.fsdecode(b"models/old\xe9/orders.sql"), _MODEL),), + expected_error="ProjectPathError", + expected_message=( + "Project path {project}/models/old\\xe9/orders.sql is not valid UTF-8; rename it " + "so SQLBuild can read it" + ), + ), + AuthoredFileFailureTestCase( + description="a unit test path that is not UTF-8 is shown lossily", + files=((os.fsdecode(b"tests/unit/caf\xe9.sql"), b'TEST (name "t");\nSELECT 1\n'),), + expected_error="ProjectPathError", + expected_message=( + "Project path {project}/tests/unit/caf\\xe9.sql is not valid UTF-8; rename it so " + "SQLBuild can read it" + ), + ), + AuthoredFileFailureTestCase( + description="a source path that is not UTF-8 is shown lossily", + files=((os.fsdecode(b"sources/caf\xe9.yml"), _SOURCE_HEAD),), + expected_error="ProjectPathError", + expected_message=( + "Project path {project}/sources/caf\\xe9.yml is not valid UTF-8; rename it so " + "SQLBuild can read it" + ), + ), + AuthoredFileFailureTestCase( + description="a model path that is not UTF-8 is shown lossily", + files=((os.fsdecode(b"models/caf\xe9.sql"), b"MODEL ();\nSELECT 1 AS id"),), + expected_error="ProjectPathError", + expected_message=( + "Project path {project}/models/caf\\xe9.sql is not valid UTF-8; rename it so " + "SQLBuild can read it" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_files_when_discovering_then_plain_yaml_loads_and_the_rest_fails_clearly( + test_case: AuthoredFileFailureTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + failure: tuple[str, str, bool] = discovery_failure_with_help(project_dir=tmp_path) + + assert failure == ( + test_case.expected_error, + test_case.expected_message.format(project=tmp_path), + test_case.expected_help, + ), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py index 7a773da3f6..f5fe19f68a 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_file_discovery.py @@ -2,28 +2,19 @@ from __future__ import annotations -import random -import unicodedata from pathlib import Path import pytest from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( EngineSwitchParityTestCase, - FactCacheFallbackTestCase, - GeneratedYamlFileParityTestCase, NativeYamlLoadTestCase, ) from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - fact_cache_keys, - generated_seed_declaration, - generated_source_document, native_yaml_tags_and_values, stage_outcome, write_project, - yaml_discovery_outcome, ) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches _SOURCES: bytes = ( b"sources:\n - name: raw_orders\n description: Orders feed.\n" @@ -38,51 +29,6 @@ ) -@pytest.mark.parametrize( - "test_case", - [ - GeneratedYamlFileParityTestCase( - description="seeded source and seed declarations in random YAML styles", - seed=81, - case_count=800, - expected_minimum_parsed=60, - expected_minimum_failed=100, - ) - ], - ids=lambda case: case.description, -) -def test_given_generated_yaml_files_when_discovering_with_each_engine_then_outcomes_match( - test_case: GeneratedYamlFileParityTestCase, tmp_path: Path -) -> None: - rng: random.Random = random.Random(test_case.seed) - files: list[tuple[tuple[str, bytes], ...]] = [ - ( - ("sources/orders.yml", generated_source_document(rng=rng).encode("utf-8")), - ("seeds/channels.yml", generated_seed_declaration(rng=rng).encode("utf-8")), - ("seeds/channels.csv", b"id,label\n1,web\n"), - ) - for _ in range(test_case.case_count) - ] - project_dirs: list[Path] = [tmp_path / f"case_{index}" for index in range(len(files))] - for project_dir, project_files in zip(project_dirs, files, strict=True): - write_project(project_dir=project_dir, files=project_files) - expected: list[object] = [ - yaml_discovery_outcome(project_dir=project_dir, native=False) - for project_dir in project_dirs - ] - - actual: list[object] = [ - yaml_discovery_outcome(project_dir=project_dir, native=True) for project_dir in project_dirs - ] - - parsed: int = sum(isinstance(outcome, str) for outcome in expected) - assert ( - mismatches(inputs=list(files), expected=expected, actual=actual), - parsed >= test_case.expected_minimum_parsed, - len(expected) - parsed >= test_case.expected_minimum_failed, - ) == (list(test_case.expected_mismatches), True, True), test_case.description - - @pytest.mark.parametrize( "test_case", [ @@ -146,73 +92,31 @@ def test_given_yaml_files_when_discovering_through_the_engine_switch_then_inputs expected_native_tags=("ok",), ), NativeYamlLoadTestCase( - description="an implicit key beyond PyYAML's simple-key limit is left to Python", + description="an implicit key beyond the simple-key limit is unsupported", files=(("sources/a.yml", f"sources: []\nmeta: {{{_BIG_HEX}: key}}\n".encode()),), - expected_native_tags=("load",), + expected_native_tags=("error",), ), NativeYamlLoadTestCase( - description="an earlier invalid file still fails before a later big integer", + description="an invalid file fails on its own and a later big integer still loads", files=( ("sources/a.yml", b"sources: [\n"), ("sources/b.yml", f"sources: []\nmeta: {{v: {_BIG_HEX}}}\n".encode()), ), - expected_native_tags=("load", "ok"), + expected_native_tags=("error", "ok"), ), ], ids=lambda case: case.description, ) -def test_given_big_integers_when_loading_natively_then_values_and_error_order_match_python( +def test_given_big_integers_when_loading_natively_then_values_match_libyaml( test_case: NativeYamlLoadTestCase, tmp_path: Path ) -> None: write_project(project_dir=tmp_path, files=test_case.files) - python: object = yaml_discovery_outcome(project_dir=tmp_path, native=False) - native: object = yaml_discovery_outcome(project_dir=tmp_path, native=True) tags, values_match = native_yaml_tags_and_values( project_dir=tmp_path, relative_paths=[path for path, _contents in test_case.files] ) - assert (native, tags, values_match) == ( - python, + assert (tags, values_match) == ( list(test_case.expected_native_tags), True, ), test_case.description - - -@pytest.mark.parametrize( - "test_case", - [ - FactCacheFallbackTestCase( - description="when native defers, Python discovery uses the fact cache as usual", - unidata_version="0.0.0", - expected_same_keys_as_python=True, - expected_native_keys=2, - ), - FactCacheFallbackTestCase( - description="native discovery itself reads no cached facts", - unidata_version=unicodedata.unidata_version, - expected_same_keys_as_python=False, - expected_native_keys=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_native_deferral_when_discovering_sources_and_tests_then_fact_cache_is_used( - test_case: FactCacheFallbackTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project( - project_dir=tmp_path, - files=( - ("sources/orders.yml", _SOURCES), - ("tests/unit/orders.sql", b'TEST (name "keeps_orders");\nSELECT 1\n'), - ), - ) - python: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=False) - monkeypatch.setattr(unicodedata, "unidata_version", test_case.unidata_version) - - native: list[tuple[str, ...]] = fact_cache_keys(project_dir=tmp_path, native=True) - - assert ((native == python), len(native)) == ( - test_case.expected_same_keys_as_python, - test_case.expected_native_keys, - ), test_case.description diff --git a/tests/unit/src/sqlbuild/cli/entry/__init__.py b/tests/unit/src/sqlbuild/cli/entry/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/entry/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/entry/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/entry/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/entry/_helpers/_test_types.py new file mode 100644 index 0000000000..74a9a780c5 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/entry/_helpers/_test_types.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class OutputStreamEncodingTestCase: + description: str + encoding: str + errors: str + text: str + expected_changed: bool + expected_bytes: bytes diff --git a/tests/unit/src/sqlbuild/cli/entry/_helpers/test_streams.py b/tests/unit/src/sqlbuild/cli/entry/_helpers/test_streams.py new file mode 100644 index 0000000000..8c31a42573 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/entry/_helpers/test_streams.py @@ -0,0 +1,55 @@ +import io + +import pytest + +from sqlbuild.cli.entry._helpers.streams import use_utf8_stream +from tests.unit.src.sqlbuild.cli.entry._helpers._test_types import OutputStreamEncodingTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + OutputStreamEncodingTestCase( + description="a cp1252 stream writes box drawing and symbols as UTF-8", + encoding="cp1252", + errors="strict", + text="\u2713 Project compiled \u2500\u2500 orders \u2192 customers\n", + expected_changed=True, + expected_bytes="\u2713 Project compiled \u2500\u2500 orders \u2192 customers\n".encode(), + ), + OutputStreamEncodingTestCase( + description="a lone surrogate is escaped instead of crashing", + encoding="cp1252", + errors="strict", + text="orders\ud800.sql\n", + expected_changed=True, + expected_bytes=b"orders\\ud800.sql\n", + ), + OutputStreamEncodingTestCase( + description="a UTF-8 stream under another spelling is left alone", + encoding="UTF8", + errors="backslashreplace", + text="\u2713 orders\n", + expected_changed=False, + expected_bytes="\u2713 orders\n".encode(), + ), + ], + ids=lambda case: case.description, +) +def test_given_stream_encoding_when_preparing_cli_output_then_any_text_is_written( + test_case: OutputStreamEncodingTestCase, +) -> None: + buffer: io.BytesIO = io.BytesIO() + stream: io.TextIOWrapper = io.TextIOWrapper( + buffer, encoding=test_case.encoding, errors=test_case.errors + ) + + changed: bool = use_utf8_stream(stream) + _ = stream.write(test_case.text) + stream.flush() + + assert (changed, buffer.getvalue()) == (test_case.expected_changed, test_case.expected_bytes) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index 200495ee9a..a1639b0983 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -5,33 +5,33 @@ import random import re from collections.abc import Callable +from functools import partial from itertools import compress from pathlib import Path +from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.discovery._helpers.filesystem import aggregation from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, - discover_model_files, discover_model_schema_files, - discover_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, - discover_test_files, -) -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - match_model_header, - match_model_headers, ) +from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files +from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_schema_files +from sqlbuild.compiler.discovery._helpers.sql.model_files import match_model_header from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, ModelHeaderMatch _DIRECTORY_NAMES: tuple[str, ...] = ( "orders", @@ -53,15 +53,19 @@ _LINK_NAMES: tuple[str, ...] = ("shared", "enums_link", "_sqlbuild_link") _PATTERNS: tuple[str, ...] = ("*", "*.sql", "*.py", "*.yml", "schema.yml", "_sqlbuild", "enums") _ISOLATED_DISCOVERERS: dict[str, Callable[..., tuple[object, ...]]] = { - "model_files": discover_model_files, + "model_files": partial( + discover_native_model_files, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + ), "enum_files": discover_enum_files, "constant_files": discover_constant_files, "model_schema_files": discover_model_schema_files, "sql_function_files": discover_sql_function_files, "sql_hook_files": discover_sql_hook_files, - "schema_files": discover_schema_files, + "schema_files": discover_native_schema_files, "seed_files": discover_seed_files, - "test_files": discover_test_files, + "test_files": discover_native_test_files, "audit_files": discover_audit_files, "macro_files": discover_macro_files, } @@ -188,13 +192,30 @@ def header_match_mismatches(*, seed: int, count: int) -> tuple[str, ...]: compress( contents, [ - native != match_model_header(value) - for value, native in zip(contents, match_model_headers(contents), strict=True) + native != _python_header_offsets(value) + for value, native in zip( + contents, _native.match_model_headers(contents), strict=True + ) ], ) ) +def _python_header_offsets(contents: str) -> tuple[int, int, int] | None: + return _mapped_offsets(match_model_header(contents)) + + +def _mapped_offsets(header_match: ModelHeaderMatch | None) -> tuple[int, int, int] | None: + return { + True: lambda: None, + False: lambda: ( + cast(ModelHeaderMatch, header_match).header_start, + cast(ModelHeaderMatch, header_match).header_end, + cast(ModelHeaderMatch, header_match).sql_start, + ), + }[header_match is None]() + + def _random_model_file(rng: random.Random) -> str: prefix: str = "".join(rng.choice(_PYTHON_WHITESPACE) for _ in range(rng.randint(0, 2))) body: str = "".join(rng.choice(_HEADER_FRAGMENTS) for _ in range(rng.randint(0, 14))) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 5d80094ac2..c863255e57 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -9,9 +9,6 @@ ) from sqlbuild.compiler.discovery._helpers.sql.audits import parse_sql_audit_file from sqlbuild.compiler.discovery._helpers.sql.hooks import parse_sql_hook_file -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_model_sql -from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file -from sqlbuild.compiler.discovery._helpers.sql.tests import parse_sql_test_file from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, @@ -23,6 +20,11 @@ from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( + parse_model_sql, + parse_sql_scenario_file, + parse_sql_test_file, +) _DECLARATION_CONTENTS_BY_KIND: dict[str, tuple[str, str]] = { "macro": (".py", "def scoped_value():\n return 1\n"), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py index fd330db8e1..743ac9f319 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_file_faults.py @@ -17,11 +17,13 @@ discover_event_exporter_declarations, discover_materialization_files, discover_provider_classes, - discover_scenario_files, - discover_source_files, discover_sql_function_files, discover_sql_hook_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, +) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import ( EventExporterDiscoveryError, ProviderDiscoveryError, @@ -38,8 +40,8 @@ "audits": discover_audit_files, "constants": discover_constant_files, "enums": discover_enum_files, - "scenarios": discover_scenario_files, - "sources": discover_source_files, + "scenarios": discover_native_scenario_files, + "sources": discover_native_source_files, "sql_functions": discover_sql_function_files, "sql_hooks": discover_sql_hook_files, } diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index 7ba642f5d8..c329b98e23 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -12,14 +12,15 @@ discover_audit_files, discover_hook_functions, discover_macro_files, - discover_model_files, discover_python_function_files, - discover_scenario_files, - discover_source_files, discover_sql_function_files, discover_sql_hook_files, - discover_test_files, ) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + discover_native_scenario_files, + discover_native_test_files, +) +from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_source_files from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( @@ -42,6 +43,7 @@ declaration_contents, discover_declarations, ) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import discover_model_files class _DiscoveredFile(Protocol): @@ -384,12 +386,12 @@ def test_given_file_in_declaration_role_when_discovering_resources_then_returns_ file_path.write_text(test_case.contents, encoding="utf-8") discoverers: dict[str, Callable[..., tuple[_DiscoveredFile, ...]]] = { "model": discover_model_files, - "test": discover_test_files, - "scenario": discover_scenario_files, + "test": discover_native_test_files, + "scenario": discover_native_scenario_files, "hook": discover_sql_hook_files, "function": discover_sql_function_files, "audit": discover_audit_files, - "source": discover_source_files, + "source": discover_native_source_files, "python_function": discover_python_function_files, "python_hook": discover_hook_functions, } diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 12c7680daa..173822217a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -1,18 +1,12 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path import pytest -from sqlbuild import _native -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_model_files +from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - model_header_column_locations, - model_output_column_locations, - parse_model_sql, -) +from sqlbuild.compiler.discovery._helpers.sql.model_files import model_output_column_locations from sqlbuild.compiler.discovery.models import ( DiscoveredSqlModelFile, DiscoveryFileFault, @@ -32,16 +26,20 @@ ParseModelSqlErrorTestCase, ParseModelSqlHeaderTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( + discover_model_files, + model_header_column_locations, + parse_model_sql, +) @pytest.mark.parametrize( "test_case", - [ExpectedCountTestCase(description="unique headers are batched", expected_count=1)], + [ExpectedCountTestCase(description="unique headers are parsed", expected_count=2)], ids=lambda case: case.description, ) -def test_given_unique_model_headers_when_discovering_then_native_tokenization_is_batched( +def test_given_unique_model_headers_when_discovering_then_header_values_and_locations_are_parsed( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, test_case: ExpectedCountTestCase, ) -> None: models_dir: Path = tmp_path / "models" @@ -54,30 +52,10 @@ def test_given_unique_model_headers_when_discovering_then_native_tokenization_is (models_dir / "second.sql").write_text( f"MODEL ({second_header});\nSELECT 2 AS total\n", encoding="utf-8" ) - native_calls: list[list[str]] = [] - native_parse: Callable[ - [list[str]], - list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]], - ] = _native.parse_model_headers - - def recording_tokenize( - headers: list[str], - ) -> list[ - tuple[ - dict[str, object] | None, - list[tuple[str, int, int]] | None, - str | None, - ] - ]: - native_calls.append(headers) - return native_parse(headers) - - monkeypatch.setattr(model_file_helpers._native, "parse_model_headers", recording_tokenize) discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files(project_dir=tmp_path) - assert len(native_calls) == test_case.expected_count - assert native_calls == [[first_header, second_header]] + assert len(discovered) == test_case.expected_count assert [model.header_values for model in discovered] == [ {"description": "one", "columns": {"café": {"type": "INTEGER"}}}, { @@ -148,8 +126,11 @@ def test_given_multiple_invalid_model_headers_when_discovering_then_fault_order_ (models_dir / "second.sql").write_text("MODEL (schema ${MISSING); SELECT 2", encoding="utf-8") faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlModelFile, ...] = discover_model_files( - project_dir=tmp_path, on_fault=faults.append + discovered: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + project_dir=tmp_path, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + on_fault=faults.append, ) assert discovered == () @@ -183,7 +164,7 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection encoding="utf-8", ) - model_file: DiscoveredSqlModelFile = discover_model_files( + model_file: DiscoveredSqlModelFile = discover_native_model_files( project_dir=tmp_path, extract_implicit_alias_columns=(test_case.expected_extract_implicit_alias_columns), extract_output_column_locations=False, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py index 701ddc0bec..4eb3533da6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSqlScenarioFileErrorTestCase, ParseSqlScenarioFileTestCase, ) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import parse_sql_scenario_file @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 66b3b7e59e..23b5edbcfc 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -3,17 +3,10 @@ import time from decimal import Decimal from pathlib import Path -from unittest.mock import Mock import pytest -from sqlbuild.compiler.discovery._helpers.filesystem import core as filesystem_core -from sqlbuild.compiler.discovery._helpers.filesystem import sql_test_files as test_file_discovery -from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers -from sqlbuild.compiler.discovery._helpers.sql.tests import ( - parse_sql_test_file, - prepare_sql_test_file_headers, -) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( omitted_ceremonial_select_offset, @@ -39,6 +32,7 @@ discovered_test_cases, discovered_test_parameters, ) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import parse_sql_test_file _FIXTURE_ROW: str = ( " SELECT 1 AS order_id, 'it''s (open)' AS note, \"Qty\" AS qty -- row (comment)\n UNION ALL\n" @@ -87,53 +81,26 @@ def test_given_large_fixture_body_when_locating_omitted_select_then_completes_wi assert min(timings) < test_case.expected_max_seconds -@pytest.mark.parametrize( - "test_case", - (ExpectedCountTestCase(description="two headers use one batch", expected_count=2),), - ids=lambda case: case.description, -) -def test_given_multiple_test_files_when_preparing_headers_then_uses_one_native_batch( - monkeypatch: pytest.MonkeyPatch, - test_case: ExpectedCountTestCase, -) -> None: - parser: Mock = Mock(wraps=model_file_helpers._native.parse_model_headers) - monkeypatch.setattr(model_file_helpers._native, "parse_model_headers", parser) - - prepare_sql_test_file_headers( - [ - 'TEST (name "native_batch_guard_alpha");\nSELECT 1', - 'TEST (name "native_batch_guard_beta");\nSELECT 2', - ] - ) - - assert parser.call_count == 1 - assert len(parser.call_args.args[0]) == test_case.expected_count - - @pytest.mark.parametrize( "test_case", (ExpectedCountTestCase(description="one invalid file reports one fault", expected_count=1),), ids=lambda case: case.description, ) -def test_given_batch_preparation_failure_when_discovering_tolerantly_then_reports_file_faults( +def test_given_invalid_test_file_when_discovering_tolerantly_then_reports_file_fault( tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, test_case: ExpectedCountTestCase, ) -> None: tests_dir: Path = tmp_path / "tests" / "unit" tests_dir.mkdir(parents=True) (tests_dir / "a_invalid.sql").write_text("SELECT 1", encoding="utf-8") (tests_dir / "b_valid.sql").write_text('TEST (name "valid_order");\nSELECT 1', encoding="utf-8") - preparation: Mock = Mock(side_effect=ValueError("invalid batch projection")) - monkeypatch.setattr(test_file_discovery, "prepare_sql_test_file_headers", preparation) faults: list[DiscoveryFileFault] = [] - discovered: tuple[DiscoveredSqlTestFile, ...] = filesystem_core.discover_test_files( + discovered: tuple[DiscoveredSqlTestFile, ...] = discover_native_test_files( project_dir=tmp_path, on_fault=faults.append, ) - assert preparation.call_count == 1 assert len(faults) == test_case.expected_count assert faults[0].path == Path("tests/unit/a_invalid.sql") discovered_names: list[str | None] = [] diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py index 5b852c051a..c6440d4e47 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py @@ -4,7 +4,6 @@ import pytest -from sqlbuild.compiler.discovery._helpers.yml.schema import parse_schema_yml from sqlbuild.spec.contracts.models import ( SchemaColumn, SchemaModelEntry, @@ -18,6 +17,7 @@ ParseSeedCsvSettingsYamlTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import expected_or_actual +from tests.unit.src.sqlbuild.compiler.discovery.helpers import parse_schema_yml @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 6ee1b29865..938711839d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -5,7 +5,6 @@ import pytest -from sqlbuild.compiler.discovery._helpers.yml.sources import parse_sources_yml from sqlbuild.compiler.discovery.exceptions import SourceParseError from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig @@ -19,6 +18,7 @@ ParseSourcesYamlTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import expected_or_actual +from tests.unit.src.sqlbuild.compiler.discovery.helpers import parse_sources_yml @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index 0ceda59449..40d07313b9 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,4 +1,38 @@ from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files +from sqlbuild.compiler.discovery._helpers.native.payloads import ( + native_locations, + native_payload_error, + native_text_runtime, +) +from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( + parse_native_sql_test_contents, +) +from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values +from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file +from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml +from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml +from sqlbuild.compiler.discovery.constants import ( + REMOVED_SQL_MODEL_HEADER_KEYS, + SQL_MODEL_HEADER_KEYS, + SQL_SCENARIO_HEADER_KEYS, + SQL_TEST_HEADER_KEYS, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, +) +from sqlbuild.compiler.discovery.types import NativeLocation +from sqlbuild.spec.contracts.models import ( + SchemaModelEntry, + SchemaSeedEntry, + SourceEntry, + SourceLocation, +) def write_lifecycle_event_sink_project(*, project_dir: Path, exporter_config: str) -> None: @@ -13,3 +47,108 @@ def write_lifecycle_event_sink_project(*, project_dir: Path, exporter_config: st "def publish(event):\n del event\n", encoding="utf-8", ) + + +def discover_model_files(*, project_dir: Path) -> tuple[DiscoveredSqlModelFile, ...]: + """Discover every model file with the column extraction full compilation uses.""" + + return discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=True, + extract_output_column_locations=True, + ) + + +def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object], str]: + """Parse model contents as model discovery does, returning header values and query SQL.""" + + payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=file_path) + return project_native_header_values(cast(dict[str, object], payload[2])), str(payload[5]) + + +def model_header_column_locations( + *, contents: str, relative_path: Path +) -> dict[str, SourceLocation]: + """Return the authored MODEL(columns) locations model discovery records.""" + + payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=relative_path) + return native_locations( + locations=cast(list[NativeLocation], payload[3]), relative_path=relative_path + ) + + +def _parsed_model(*, contents: str, file_path: Path) -> tuple[object, ...]: + payload: tuple[object, ...] = _native.parse_model_contents( + { + "file_path": str(file_path), + "supported_keys": sorted(SQL_MODEL_HEADER_KEYS), + "removed_keys": sorted(REMOVED_SQL_MODEL_HEADER_KEYS), + "extract_implicit_alias_columns": True, + "extract_output_column_locations": False, + **native_text_runtime(), + }, + contents, + ) + _raise_payload_error(payload=payload, file_path=file_path) + return payload + + +def parse_sql_test_file(*, contents: str, file_path: Path) -> tuple[DiscoveredSqlTestBlock, ...]: + """Parse SQL test contents as test discovery does.""" + + return parse_native_sql_test_contents(contents=contents, file_path=file_path) + + +def parse_sql_scenario_file( + *, contents: str, file_path: Path, relative_path: Path +) -> DiscoveredSqlScenarioFile: + """Parse scenario contents as scenario discovery does.""" + + payload: tuple[object, ...] = _native.parse_scenario_contents( + { + "file_path": str(file_path), + "test_keys": sorted(SQL_TEST_HEADER_KEYS), + "scenario_keys": sorted(SQL_SCENARIO_HEADER_KEYS), + **native_text_runtime(), + }, + contents, + ) + _raise_payload_error(payload=payload, file_path=file_path) + return build_sql_scenario_file( + header_values=project_native_header_values(cast(dict[str, object], payload[2])), + sql_body=str(payload[3]), + contents=str(payload[1]), + file_path=file_path, + relative_path=relative_path, + ) + + +def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, ...]: + """Load and parse sources YAML contents as source discovery does.""" + + return parse_loaded_sources_yml( + loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="source"), + file_path=file_path, + ) + + +def parse_schema_yml( + *, contents: str, file_path: Path +) -> tuple[tuple[SchemaModelEntry, ...], tuple[SchemaSeedEntry, ...]]: + """Load and parse schema or seed YAML contents as schema discovery does.""" + + return parse_loaded_schema_yml( + loaded=_loaded_yaml(contents=contents, file_path=file_path, kind="schema"), + file_path=file_path, + ) + + +def _loaded_yaml(*, contents: str, file_path: Path, kind: str) -> object: + payload: tuple[object, ...] = _native.load_yaml_document(str(file_path), contents, kind) + _raise_payload_error(payload=payload, file_path=file_path) + return payload[2] + + +def _raise_payload_error(*, payload: tuple[object, ...], file_path: Path) -> None: + for error in filter(None, (native_payload_error(payload=payload, file_path=file_path),)): + raise error diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py index 0c5be7341e..8a9cdd162b 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py @@ -6,12 +6,14 @@ import pytest -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_model_sql -from sqlbuild.compiler.discovery._helpers.sql.scenarios import parse_sql_scenario_file from sqlbuild.compiler.discovery.exceptions import SqlScenarioParseError from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import format_native_headers, lint_native_headers from sqlbuild.lint.models import LintConfig +from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( + parse_model_sql, + parse_sql_scenario_file, +) from tests.unit.src.sqlbuild.lint._helpers._test_types import ( FormatDescriptionTestCase, FormatNativeTestCase, diff --git a/uv.lock b/uv.lock index 6d6d078cab..43b824ac31 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.12" +requires-python = ">=3.12, <3.15" resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten')", diff --git a/website/src/content/docs/docs/concepts/resource-identities.mdx b/website/src/content/docs/docs/concepts/resource-identities.mdx index 4510522e4f..1a0f59282d 100644 --- a/website/src/content/docs/docs/concepts/resource-identities.mdx +++ b/website/src/content/docs/docs/concepts/resource-identities.mdx @@ -37,6 +37,13 @@ error[D016]: Invalid model identity 'DailyOrders' in models/DailyOrders.sql; use snake_case 'daily_orders' ``` +Discovery also fails with `D016` when the project directory, or a model, unit test, scenario, source +or schema YAML file it reads, has a path that is not valid UTF-8, including a directory above the +file. The error shows each invalid byte as a `\xNN` escape; rename the path to valid UTF-8. Other +files with such names, such as notes or data files, and directories that hold none of those files +are ignored. Commands that tolerate unreadable files, such as `sqb scope`, report the file and +continue. + SQLBuild suggests a corrected spelling but never silently normalizes an identity. Silent normalization would make selectors, manifests, persisted execution state, and integration keys disagree about which resource ran. diff --git a/website/src/content/docs/docs/concepts/sources.mdx b/website/src/content/docs/docs/concepts/sources.mdx index efd123df59..51abce14df 100644 --- a/website/src/content/docs/docs/concepts/sources.mdx +++ b/website/src/content/docs/docs/concepts/sources.mdx @@ -284,3 +284,25 @@ Use [`sqb freshness`](/docs/cli/freshness) to observe source freshness on demand A sources file contains only `sources` and `dlt_sources`. Unknown keys in a sources file, a source, a source column, `freshness`, `freshness.age_policy`, or `ingestr` fail discovery with `D006`, naming the key and suggesting the nearest supported key. + +## YAML that SQLBuild reads + +Sources files, seed declarations and `schema.yml` files use plain YAML 1.1: block and flow mappings +and sequences, plain and quoted scalars, literal (`|`) and folded (`>`) block scalars, comments, +anchors, aliases and merge keys (`<<`). Files must be UTF-8. + +Invalid YAML fails discovery with `D006` (sources) or `D005` (seed declarations and `schema.yml`), +naming the file, line and column. The following YAML features also fail with the same codes and a +hint to rewrite the value in plain YAML: + +- tab characters anywhere in the file, and byte order marks after its start +- `%YAML` and `%TAG` directives +- the `!!binary`, `!!set`, `!!omap` and `!!pairs` tags, and an empty scalar tagged `!` +- recursive aliases, and aliases that expand beyond SQLBuild's size limit +- nesting deeper than 256 levels, integers longer than 4300 digits, and implicit keys longer than + 1000 characters +- NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR line breaks +- `?` inside a plain scalar in a flow collection, and `:` directly before `,`, `]` or `}` in a + flow collection +- a block scalar used as a key, inside a flow collection, or with indentation SQLBuild cannot read + unambiguously diff --git a/website/src/content/docs/docs/quickstart.mdx b/website/src/content/docs/docs/quickstart.mdx index a1da26990a..652ec3f188 100644 --- a/website/src/content/docs/docs/quickstart.mdx +++ b/website/src/content/docs/docs/quickstart.mdx @@ -7,7 +7,7 @@ This guide walks you through creating and running a complete transformation proj ## Prerequisites -- Python 3.12+ +- Python 3.12–3.14 (other versions stop with `D017` naming the supported versions) - SQLBuild installed: `uv pip install sqlbuild` or `pip install sqlbuild` Confirm the installed release with `sqb --version`.