diff --git a/crates/sqlbuild-config/src/toml/_helpers/conversion.rs b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs index 5486a937ad..08310074da 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/conversion.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs @@ -156,16 +156,30 @@ fn table(entries: &Table, depth: usize) -> Result { Ok(ordered_map(converted)) } +/// The one-based line and character column of byte `offset` in `text`. +pub(crate) fn line_and_column(text: &str, offset: usize) -> (usize, usize) { + let before = text.get(..offset).unwrap_or(text); + let line_start = before.rfind('\n').map_or(0, |index| index + 1); + ( + before.matches('\n').count() + 1, + before[line_start..].chars().count() + 1, + ) +} + +/// A syntax error located at byte `offset` of `text`. +pub(crate) fn syntax_error(text: &str, offset: usize, message: impl Into) -> ConfigError { + let (line, column) = line_and_column(text, offset); + ConfigError::new(ConfigErrorKind::Syntax, message).at(line, column) +} + /// Parse `text` and convert its root table. pub(crate) fn parse_document(text: &str) -> Result { let document = Document::parse(text).map_err(|error| { - let offset = error.span().map_or(0, |span| span.start); - ConfigError { - kind: ConfigErrorKind::Syntax, - message: error.message().to_owned(), - line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), - column: None, - } + syntax_error( + text, + error.span().map_or(0, |span| span.start), + error.message(), + ) })?; table(document.as_table(), 0).map(|(converted, _)| converted) } diff --git a/crates/sqlbuild-config/src/toml/_helpers/integers.rs b/crates/sqlbuild-config/src/toml/_helpers/integers.rs deleted file mode 100644 index 06ca798e0a..0000000000 --- a/crates/sqlbuild-config/src/toml/_helpers/integers.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! Values the TOML parser rejects but `tomllib` reads: integers beyond 64 bits and deep nesting. - -use crate::errors::{ConfigError, ConfigErrorKind}; -use crate::models::{ConfigValue, Natural}; - -const DECIMAL_RADIX: u32 = 10; -/// Noncharacters that can mark a placeholder; the first absent from a document is used. -const PLACEHOLDER_MARKS: std::ops::RangeInclusive = 0xFDD0..=0xFDEF; - -/// A value read apart from the document: its byte range there and the value it stands for. -pub(crate) type Placeholder = (usize, usize, ConfigValue); - -fn digits_with_separators(body: &str, radix: u32) -> Option> { - let mut values: Vec = Vec::new(); - let mut after_digit = false; - for character in body.chars() { - if character == '_' && after_digit { - after_digit = false; - continue; - } - values.push(character.to_digit(radix)?); - after_digit = true; - } - after_digit.then_some(values) -} - -fn natural(values: &[u32], radix: u32) -> String { - values - .iter() - .fold(Natural::default(), |total, value| { - total.times_plus(radix, *value) - }) - .decimal() -} - -/// The signed decimal digits of a TOML integer literal outside the 64-bit range, if `raw` is one. -pub(crate) fn large_integer(raw: &str) -> Option { - let (radix, body, negative) = match raw.get(..2) { - Some("0x") => (16, &raw[2..], false), - Some("0o") => (8, &raw[2..], false), - Some("0b") => (2, &raw[2..], false), - _ => match raw.strip_prefix('-') { - Some(rest) => (10, rest, true), - None => (10, raw.strip_prefix('+').unwrap_or(raw), false), - }, - }; - let values = digits_with_separators(body, radix)?; - if radix == DECIMAL_RADIX && values.len() > 1 && values[0] == 0 { - return None; - } - let digits = natural(&values, radix); - let signed = if negative { - format!("-{digits}") - } else { - digits - }; - signed.parse::().is_err().then_some(signed) -} - -/// A placeholder mark that neither appears in `text` nor can come from one of its escapes. -pub(crate) fn placeholder_mark(text: &str) -> Result { - let lowered = text.to_ascii_lowercase(); - PLACEHOLDER_MARKS - .filter_map(char::from_u32) - .find(|mark| !text.contains(*mark) && !lowered.contains(&format!("{:x}", u32::from(*mark)))) - .ok_or_else(|| { - ConfigError::new( - ConfigErrorKind::Unsupported, - "large integers in text using every placeholder mark are not read natively", - ) - }) -} - -/// `text` with each placeholder range replaced by a string marked by `mark`. -pub(crate) fn substitute_placeholders(text: &str, integers: &[Placeholder], mark: char) -> String { - let mut substituted = String::with_capacity(text.len()); - let mut copied = 0; - for (index, (start, end, _)) in integers.iter().enumerate() { - substituted.push_str(&text[copied..*start]); - substituted.push_str(&format!("\"{mark}{index}\"")); - copied = *end; - } - substituted.push_str(&text[copied..]); - substituted -} - -/// `value` with each placeholder string turned back into the value it stands for. -pub(crate) fn restore_placeholders( - value: ConfigValue, - integers: &[Placeholder], - mark: char, -) -> ConfigValue { - match value { - ConfigValue::String(text) => match text - .strip_prefix(mark) - .and_then(|index| { - integers - .iter() - .enumerate() - .find(|(position, _)| position.to_string() == index) - }) - .map(|(_, integer)| integer) - { - Some((_, _, restored)) => restored.clone(), - None => ConfigValue::String(text), - }, - ConfigValue::List(items) => ConfigValue::List( - items - .into_iter() - .map(|item| restore_placeholders(item, integers, mark)) - .collect(), - ), - ConfigValue::Map(entries) => ConfigValue::Map( - entries - .into_iter() - .map(|(key, item)| (key, restore_placeholders(item, integers, mark))) - .collect(), - ), - other => other, - } -} diff --git a/crates/sqlbuild-config/src/toml/_helpers/mod.rs b/crates/sqlbuild-config/src/toml/_helpers/mod.rs index eacec567c1..419b7e0a9a 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/mod.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/mod.rs @@ -1,5 +1,3 @@ pub(crate) mod conversion; -pub(crate) mod integers; -pub(crate) mod namespaces; pub(crate) mod reading; pub(crate) mod version; diff --git a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs deleted file mode 100644 index afecd22a49..0000000000 --- a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs +++ /dev/null @@ -1,342 +0,0 @@ -//! `tomllib`'s namespace rules: which tables, dotted keys and arrays of tables may be (re)opened. - -use crate::errors::{ConfigError, ConfigErrorKind}; -use crate::toml::constants::MAX_CONTAINER_DEPTH; -use std::collections::HashMap; -use toml_parser::decoder::Encoding; -use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; -use toml_parser::{ErrorSink, ParseError, Raw, Source, Span}; - -/// `Flags.FROZEN`: a value written with `=`, which later statements cannot extend. -const FROZEN: u8 = 1; -/// `Flags.EXPLICIT_NEST`: a table declared by a header or opened by a dotted key. -const EXPLICIT_NEST: u8 = 2; -/// One key of `tomllib`'s `Flags` tree. -#[derive(Default)] -struct FlagNode { - flags: u8, - recursive: u8, - nested: HashMap, -} - -/// `tomllib`'s `Flags`, keyed by table names without array indices. -#[derive(Default)] -struct Flags { - root: HashMap, - pending: Vec>, -} - -impl Flags { - fn is(&self, key: &[String], flag: u8) -> bool { - let Some((stem, parent)) = key.split_last() else { - return false; - }; - let mut container = &self.root; - for name in parent { - let Some(inner) = container.get(name) else { - return false; - }; - if inner.recursive & flag != 0 { - return true; - } - container = &inner.nested; - } - container - .get(stem) - .is_some_and(|node| (node.flags | node.recursive) & flag != 0) - } - - fn set(&mut self, key: &[String], flag: u8, recursive: bool) { - let Some((stem, parent)) = key.split_last() else { - return; - }; - let mut container = &mut self.root; - for name in parent { - container = &mut container.entry(name.clone()).or_default().nested; - } - let node = container.entry(stem.clone()).or_default(); - match recursive { - true => node.recursive |= flag, - false => node.flags |= flag, - } - } - - fn unset_all(&mut self, key: &[String]) { - let Some((stem, parent)) = key.split_last() else { - return; - }; - let mut container = &mut self.root; - for name in parent { - let Some(inner) = container.get_mut(name) else { - return; - }; - container = &mut inner.nested; - } - container.remove(stem); - } - - fn finalize_pending(&mut self) { - for key in std::mem::take(&mut self.pending) { - self.set(&key, EXPLICIT_NEST, false); - } - } -} - -/// What `tomllib`'s `NestedDict` holds behind one key. -enum Entry { - Table(HashMap), - Tables(Vec>), - Value, -} - -/// `tomllib`'s `NestedDict`: the tables and values created so far. -#[derive(Default)] -struct NestedDict { - root: HashMap, -} - -impl NestedDict { - /// `get_or_create_nest`; `None` where `tomllib` raises "There is no nest behind this key". - fn nest(&mut self, key: &[String], access_lists: bool) -> Option<&mut HashMap> { - let mut container = &mut self.root; - for name in key { - let entry = container - .entry(name.clone()) - .or_insert_with(|| Entry::Table(HashMap::new())); - container = match entry { - Entry::Table(table) => table, - Entry::Tables(tables) if access_lists => tables.last_mut()?, - _ => return None, - }; - } - Some(container) - } -} - -/// The keys and values of one inline table, checked as `tomllib`'s `parse_inline_table` does. -#[derive(Default)] -struct InlineTable { - data: NestedDict, - flags: Flags, - key: Vec, -} - -impl InlineTable { - fn insert(&mut self, container_value: bool) -> bool { - let key = std::mem::take(&mut self.key); - let Some((stem, parent)) = key.split_last() else { - return false; - }; - if self.flags.is(&key, FROZEN) { - return false; - } - let Some(table) = self.data.nest(parent, false) else { - return false; - }; - if table.contains_key(stem) { - return false; - } - table.insert(stem.clone(), Entry::Value); - if container_value { - self.flags.set(&key, FROZEN, true); - } - true - } -} - -/// An open array or inline table inside a value. -enum Frame { - Array, - InlineTable(InlineTable), -} - -/// Replays `tomllib.loads`' bookkeeping over parser events and records the first rejection. -struct NamespaceReceiver<'text> { - source: Source<'text>, - data: NestedDict, - flags: Flags, - header: Vec, - key: Vec, - frames: Vec, - rejected: Option<(usize, &'static str)>, -} - -impl NamespaceReceiver<'_> { - fn reject(&mut self, span: Span, reason: &'static str) { - self.rejected.get_or_insert((span.start(), reason)); - } - - fn decode(&self, span: Span, encoding: Option) -> String { - let raw = self.source.get(span).map_or("", |raw| raw.as_str()); - let mut decoded = String::new(); - let mut ignored: Vec = Vec::new(); - Raw::new_unchecked(raw, encoding, span).decode_key(&mut decoded, &mut ignored); - decoded - } - - /// `key_value_rule` for a key-value pair under the current header. - fn key_value(&mut self, span: Span, container_value: bool) { - let key = std::mem::take(&mut self.key); - let Some((stem, parent)) = key.split_last() else { - return; - }; - for length in 1..key.len() { - let container: Vec = [&self.header[..], &key[..length]].concat(); - if self.flags.is(&container, EXPLICIT_NEST) { - return self.reject(span, "Cannot redefine namespace"); - } - self.flags.pending.push(container); - } - let parent_key: Vec = [&self.header[..], parent].concat(); - if self.flags.is(&parent_key, FROZEN) { - return self.reject(span, "Cannot mutate immutable namespace"); - } - let Some(table) = self.data.nest(&parent_key, true) else { - return self.reject(span, "Cannot overwrite a value"); - }; - if table.contains_key(stem) { - return self.reject(span, "Cannot overwrite a value"); - } - table.insert(stem.clone(), Entry::Value); - if container_value { - let full_key: Vec = [&self.header[..], &key[..]].concat(); - self.flags.set(&full_key, FROZEN, true); - } - } - - /// `create_dict_rule` for a `[table]` header. - fn table(&mut self, span: Span) { - let key = std::mem::take(&mut self.key); - if self.flags.is(&key, EXPLICIT_NEST) || self.flags.is(&key, FROZEN) { - return self.reject(span, "Cannot declare a table twice"); - } - self.flags.set(&key, EXPLICIT_NEST, false); - if self.data.nest(&key, true).is_none() { - return self.reject(span, "Cannot overwrite a value"); - } - self.header = key; - } - - /// `create_list_rule` for an `[[array of tables]]` header. - fn array_of_tables(&mut self, span: Span) { - let key = std::mem::take(&mut self.key); - let Some((stem, parent)) = key.split_last() else { - return; - }; - if self.flags.is(&key, FROZEN) { - return self.reject(span, "Cannot mutate immutable namespace"); - } - self.flags.unset_all(&key); - self.flags.set(&key, EXPLICIT_NEST, false); - let Some(table) = self.data.nest(parent, true) else { - return self.reject(span, "Cannot overwrite a value"); - }; - match table.get_mut(stem) { - None => { - table.insert(stem.clone(), Entry::Tables(vec![HashMap::new()])); - } - Some(Entry::Tables(tables)) => tables.push(HashMap::new()), - Some(_) => return self.reject(span, "Cannot overwrite a value"), - } - self.header = key; - } - - /// A complete value: a scalar, or an array or inline table that just closed. - fn value(&mut self, span: Span, container_value: bool) { - match self.frames.last_mut() { - None => self.key_value(span, container_value), - Some(Frame::Array) => {} - Some(Frame::InlineTable(table)) => { - if !table.insert(container_value) { - self.reject(span, "Cannot redefine an inline table key"); - } - } - } - } - - fn open_header(&mut self) { - self.flags.finalize_pending(); - self.key.clear(); - } -} - -impl EventReceiver for NamespaceReceiver<'_> { - fn std_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { - self.open_header(); - } - - fn std_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { - self.table(span); - } - - fn array_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { - self.open_header(); - } - - fn array_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { - self.array_of_tables(span); - } - - fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { - self.frames.push(Frame::InlineTable(InlineTable::default())); - true - } - - fn inline_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { - self.frames.pop(); - self.value(span, true); - } - - fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { - self.frames.push(Frame::Array); - true - } - - fn array_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { - self.frames.pop(); - self.value(span, true); - } - - fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { - let segment = self.decode(span, encoding); - match self.frames.last_mut() { - Some(Frame::InlineTable(table)) => table.key.push(segment), - Some(Frame::Array) => {} - None => self.key.push(segment), - } - } - - fn scalar(&mut self, span: Span, _encoding: Option, _error: &mut dyn ErrorSink) { - self.value(span, false); - } -} - -/// Reject what `tomllib` rejects as a redefined or immutable namespace. -pub(crate) fn reject_namespace_conflicts(text: &str) -> Result<(), ConfigError> { - let source = Source::new(text); - let tokens = source.lex().into_vec(); - let mut receiver = NamespaceReceiver { - source, - data: NestedDict::default(), - flags: Flags::default(), - header: Vec::new(), - key: Vec::new(), - frames: Vec::new(), - rejected: None, - }; - let mut errors: Vec = Vec::new(); - parse_document( - &tokens, - &mut RecursionGuard::new(&mut receiver, MAX_CONTAINER_DEPTH), - &mut errors, - ); - match receiver.rejected { - None => Ok(()), - Some((offset, reason)) => Err(ConfigError { - kind: ConfigErrorKind::Syntax, - message: reason.to_owned(), - line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), - column: None, - }), - } -} diff --git a/crates/sqlbuild-config/src/toml/_helpers/reading.rs b/crates/sqlbuild-config/src/toml/_helpers/reading.rs index abb2cea2ee..04a7fa1014 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/reading.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/reading.rs @@ -1,78 +1,17 @@ -//! Read a TOML document natively, splitting off what the parser cannot nest. +//! Read a TOML 1.0 document natively. use crate::constants::BYTE_ORDER_MARK; -use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::errors::ConfigError; use crate::models::ConfigValue; -use crate::toml::_helpers::integers::{ - Placeholder, placeholder_mark, restore_placeholders, substitute_placeholders, -}; -use crate::toml::constants::SPLIT_VALUE_KEY; +use crate::toml::_helpers::conversion::parse_document; +use crate::toml::_helpers::version::reject_toml_1_1_syntax; -/// The value of one container nested past the parser's limit, read as its own document. -fn deep_value(text: &str, (start, end): (usize, usize)) -> Result { - let document = format!("{SPLIT_VALUE_KEY} = {}", &text[start..end]); - let lines_before = text[..start].matches('\n').count(); - let value = load_document(&document).map_err(|error| ConfigError { - line: error.line.map(|line| line + lines_before), - ..error - })?; - let restored = value - .get(SPLIT_VALUE_KEY) - .cloned() - .unwrap_or(ConfigValue::Null); - Ok((start, end, restored)) -} - -fn is_inside(ranges: &[(usize, usize)], offset: usize) -> bool { - ranges - .iter() - .any(|(start, end)| *start <= offset && offset < *end) -} - -/// An upper bound of the bracket nesting of `text`, counting brackets inside strings too. -pub(crate) fn bracket_depth(text: &str) -> usize { - let mut depth: usize = 0; - let mut deepest: usize = 0; - for byte in text.bytes() { - match byte { - b'[' | b'{' => { - depth += 1; - deepest = deepest.max(depth); - } - b']' | b'}' => depth = depth.saturating_sub(1), - _ => {} - } - } - deepest -} - -/// `tomllib.loads` on the current thread. +/// The document's root table; a leading byte order mark is ignored, as editors add one. pub(crate) fn load_document(text: &str) -> Result { - if text.starts_with(BYTE_ORDER_MARK) { - return Err(ConfigError::new( - ConfigErrorKind::Syntax, - "invalid statement: tomllib does not skip a byte order mark", - )); - } - let text = text.replace("\r\n", "\n"); - let (large_integers, deep_ranges) = - crate::toml::_helpers::version::reject_toml_1_1_syntax(&text)?; - crate::toml::_helpers::namespaces::reject_namespace_conflicts(&text)?; - let mut placeholders: Vec = deep_ranges - .iter() - .map(|range| deep_value(&text, *range)) - .collect::>()?; - placeholders.extend( - large_integers - .into_iter() - .filter(|(start, _, _)| !is_inside(&deep_ranges, *start)), - ); - if placeholders.is_empty() { - return crate::toml::_helpers::conversion::parse_document(&text); - } - placeholders.sort_by_key(|(start, _, _)| *start); - let mark = placeholder_mark(&text)?; - let substituted = substitute_placeholders(&text, &placeholders, mark); - crate::toml::_helpers::conversion::parse_document(&substituted) - .map(|value| restore_placeholders(value, &placeholders, mark)) + let text = text + .strip_prefix(BYTE_ORDER_MARK) + .unwrap_or(text) + .replace("\r\n", "\n"); + reject_toml_1_1_syntax(&text)?; + parse_document(&text) } diff --git a/crates/sqlbuild-config/src/toml/_helpers/version.rs b/crates/sqlbuild-config/src/toml/_helpers/version.rs index 2d13536ef7..7b69e62e34 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/version.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/version.rs @@ -1,16 +1,12 @@ -//! Find TOML 1.1 syntax, which the parser accepts and Python's TOML 1.0 `tomllib` rejects. +//! Reject TOML 1.1 syntax, which the parser accepts but the Python commands' TOML 1.0 reader rejects. -use crate::errors::{ConfigError, ConfigErrorKind}; -use crate::models::ConfigValue; -use crate::toml::_helpers::integers::{Placeholder, large_integer}; -use crate::toml::constants::{MAX_CONTAINER_DEPTH, SPLIT_CONTAINER_DEPTH}; +use crate::errors::ConfigError; +use crate::toml::_helpers::conversion::syntax_error; +use crate::toml::constants::MAX_CONTAINER_DEPTH; use toml_parser::decoder::Encoding; use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; use toml_parser::{ErrorSink, ParseError, Source, Span}; -/// Large integers, and the byte ranges of containers nested at the split depth. -pub(crate) type NativeSplits = (Vec, Vec<(usize, usize)>); - /// The most dotted key segments read natively, which bounds the table nesting of a document. const MAX_KEY_SEGMENTS: usize = 128; @@ -28,9 +24,6 @@ struct VersionReceiver<'text> { after_comma: bool, key_segments: usize, found: Option<(usize, &'static str)>, - large_integers: Vec, - deep_values: Vec<(usize, usize)>, - deep_start: usize, } impl VersionReceiver<'_> { @@ -57,29 +50,11 @@ impl VersionReceiver<'_> { } } - fn record_large_integer(&mut self, span: Span) { - let raw = self - .source - .get(span) - .map(|raw| raw.as_str()) - .unwrap_or_default(); - if let Some(digits) = large_integer(raw) { - self.large_integers - .push((span.start(), span.end(), ConfigValue::BigInteger(digits))); - } - } - - fn open_container(&mut self, span: Span, container: Container) { + fn open_container(&mut self, container: Container) { self.containers.push(container); - if self.containers.len() == SPLIT_CONTAINER_DEPTH { - self.deep_start = span.start(); - } } - fn close_container(&mut self, span: Span) { - if self.containers.len() == SPLIT_CONTAINER_DEPTH { - self.deep_values.push((self.deep_start, span.end())); - } + fn close_container(&mut self) { self.containers.pop(); } @@ -115,9 +90,9 @@ impl EventReceiver for VersionReceiver<'_> { self.end_key(); } - fn inline_table_open(&mut self, span: Span, _error: &mut dyn ErrorSink) -> bool { + fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { self.end_key(); - self.open_container(span, Container::InlineTable); + self.open_container(Container::InlineTable); true } @@ -126,18 +101,18 @@ impl EventReceiver for VersionReceiver<'_> { self.record(span, "a trailing comma in an inline table"); } self.token(); - self.close_container(span); + self.close_container(); } - fn array_open(&mut self, span: Span, _error: &mut dyn ErrorSink) -> bool { + fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { self.token(); - self.open_container(span, Container::Array); + self.open_container(Container::Array); true } - fn array_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + fn array_close(&mut self, _span: Span, _error: &mut dyn ErrorSink) { self.token(); - self.close_container(span); + self.close_container(); } fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { @@ -160,9 +135,6 @@ impl EventReceiver for VersionReceiver<'_> { fn scalar(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { self.token(); self.check_escapes(span, encoding); - if encoding.is_none() { - self.record_large_integer(span); - } } fn value_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { @@ -180,8 +152,8 @@ impl EventReceiver for VersionReceiver<'_> { } } -/// Reject TOML 1.1 and syntax errors; return large integers and values nested past the parser. -pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result { +/// Reject TOML 1.1 syntax, syntax errors and nesting past the supported depth. +pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { let source = Source::new(text); let tokens = source.lex().into_vec(); let mut receiver = VersionReceiver { @@ -190,9 +162,6 @@ pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result = Vec::new(); parse_document( @@ -202,20 +171,14 @@ pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result Ok((receiver.large_integers, receiver.deep_values)), - Some((offset, construct)) => Err(ConfigError { - kind: ConfigErrorKind::Syntax, - message: format!("{construct} is TOML 1.1 syntax, which tomllib rejects"), - line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), - column: None, - }), + None => Ok(()), + Some((offset, construct)) => Err(syntax_error( + text, + offset, + format!("{construct} is TOML 1.1 syntax; sqlbuild config files use TOML 1.0"), + )), } } diff --git a/crates/sqlbuild-config/src/toml/constants.rs b/crates/sqlbuild-config/src/toml/constants.rs index 24f1b965a6..8eae738285 100644 --- a/crates/sqlbuild-config/src/toml/constants.rs +++ b/crates/sqlbuild-config/src/toml/constants.rs @@ -1,12 +1,4 @@ //! Native TOML nesting limits. -/// The deepest inline nesting read natively; `tomllib` exhausts Python's recursion limit by 497. -pub(crate) const MAX_CONTAINER_DEPTH: u32 = 512; -/// Inline containers at this depth are read as separate values, under the parser's own limit. -pub(crate) const SPLIT_CONTAINER_DEPTH: usize = 64; -/// The key a separately read value is assigned to. -pub(crate) const SPLIT_VALUE_KEY: &str = "value"; -/// Bracket depth past which a document is read on a thread with a larger stack. -pub(crate) const DEEP_DOCUMENT_BRACKETS: usize = 128; -/// The stack of that thread: the recursive parser needs more than a default thread has. -pub(crate) const DEEP_DOCUMENT_STACK_BYTES: usize = 64 * 1024 * 1024; +/// The deepest container nesting a config file may use; deeper documents are rejected. +pub(crate) const MAX_CONTAINER_DEPTH: u32 = 64; diff --git a/crates/sqlbuild-config/src/toml/main/load_toml.rs b/crates/sqlbuild-config/src/toml/main/load_toml.rs index 7f0120c8c2..70035e8310 100644 --- a/crates/sqlbuild-config/src/toml/main/load_toml.rs +++ b/crates/sqlbuild-config/src/toml/main/load_toml.rs @@ -1,26 +1,10 @@ -//! Load a TOML document the way Python's `tomllib.loads` does. +//! Load a TOML 1.0 document natively. -use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::errors::ConfigError; use crate::models::ConfigValue; -use crate::toml::_helpers::reading::{bracket_depth, load_document}; -use crate::toml::constants::{DEEP_DOCUMENT_BRACKETS, DEEP_DOCUMENT_STACK_BYTES}; +use crate::toml::_helpers::reading::load_document; -/// The table `tomllib.loads(text)` returns, rejecting TOML 1.1 syntax as `tomllib` does. +/// The document's root table, rejecting TOML 1.1 syntax and nesting past the supported depth. pub fn load_toml(text: &str) -> Result { - if bracket_depth(text) <= DEEP_DOCUMENT_BRACKETS { - return load_document(text); - } - std::thread::scope(|scope| { - std::thread::Builder::new() - .stack_size(DEEP_DOCUMENT_STACK_BYTES) - .spawn_scoped(scope, || load_document(text)) - .map_err(|error| ConfigError::new(ConfigErrorKind::Unsupported, error.to_string()))? - .join() - .unwrap_or_else(|_| { - Err(ConfigError::new( - ConfigErrorKind::Unsupported, - "the deep TOML reader stopped unexpectedly", - )) - }) - }) + load_document(text) } diff --git a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs index b32405b423..6beb6172be 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs @@ -2,7 +2,50 @@ use crate::errors::ConfigErrorKind; use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; use crate::toml::main::load_toml::load_toml; use crate::toml::tests::helpers::table; -use crate::toml::tests::test_types::{DeepTomlTestCase, HostileTomlTestCase, LoadTomlTestCase}; +use crate::toml::tests::test_types::{ + DeepTomlTestCase, HostileTomlTestCase, LoadTomlTestCase, TomlErrorPositionTestCase, +}; + +#[test] +fn given_invalid_toml_when_loading_then_error_reports_line_and_character_column() { + let test_cases = [ + TomlErrorPositionTestCase { + description: "an unclosed table header points into the header line", + text: "name = \"orders\"\n\n[settings\nsql_analysis = true\n".to_owned(), + expected_position: (Some(3), Some(10)), + }, + TomlErrorPositionTestCase { + description: "a leading byte order mark does not shift first-line columns", + text: "\u{feff}a = 1 x".to_owned(), + expected_position: (Some(1), Some(5)), + }, + TomlErrorPositionTestCase { + description: "the same error without a byte order mark", + text: "a = 1 x".to_owned(), + expected_position: (Some(1), Some(5)), + }, + TomlErrorPositionTestCase { + description: "columns count characters, not bytes, after multibyte text", + text: "label = \"café\" x\n".to_owned(), + expected_position: (Some(1), Some(16)), + }, + TomlErrorPositionTestCase { + description: "nesting past the supported depth points at the container exceeding it", + text: format!("x = 1\nvalue = {}1{}\n", "[".repeat(70), "]".repeat(70)), + expected_position: (Some(2), Some(73)), + }, + ]; + for test_case in test_cases { + let error = load_toml(&test_case.text).expect_err(test_case.description); + assert_eq!( + (error.line, error.column), + test_case.expected_position, + "{}: {}", + test_case.description, + error.message + ); + } +} #[test] fn given_toml_documents_when_loading_then_values_match_tomllib() { @@ -29,35 +72,16 @@ fn given_toml_documents_when_loading_then_values_match_tomllib() { ])), }, LoadTomlTestCase { - description: "integers beyond 64 bits are read exactly in every base and position", - text: "a = 18_446_744_073_709_551_616\nb = -9223372036854775809\n\ - c = [0xFFFF_FFFF_FFFF_FFFF, 0o7777777777777777777777]\nd = {e = 0b1_0000000000000000000000000000000000000000000000000000000000000000}\n\ - f = '\u{fdd0}0'\ng = 9223372036854775807", + description: "integers beyond 64 bits are rejected", + text: "a = 18_446_744_073_709_551_616", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "the 64-bit integer bounds are read exactly", + text: "a = 9223372036854775807\nb = -9223372036854775808", expected_value: Ok(table(vec![ - ( - "a", - ConfigValue::BigInteger("18446744073709551616".to_owned()), - ), - ( - "b", - ConfigValue::BigInteger("-9223372036854775809".to_owned()), - ), - ( - "c", - ConfigValue::List(vec![ - ConfigValue::BigInteger("18446744073709551615".to_owned()), - ConfigValue::BigInteger("73786976294838206463".to_owned()), - ]), - ), - ( - "d", - table(vec![( - "e", - ConfigValue::BigInteger("18446744073709551616".to_owned()), - )]), - ), - ("f", ConfigValue::String("\u{fdd0}0".to_owned())), - ("g", ConfigValue::Integer(i64::MAX)), + ("a", ConfigValue::Integer(i64::MAX)), + ("b", ConfigValue::Integer(i64::MIN)), ])), }, LoadTomlTestCase { @@ -246,9 +270,18 @@ fn given_toml_documents_when_loading_then_values_match_tomllib() { ])), }, LoadTomlTestCase { - description: "a dotted key cannot reach into an array of tables", + description: "a dotted key under a table extends its array of tables' last element", text: "[[a.b]]\n[a]\nb.c.d = 1", - expected_value: Err(ConfigErrorKind::Syntax), + expected_value: Ok(table(vec![( + "a", + table(vec![( + "b", + ConfigValue::List(vec![table(vec![( + "c", + table(vec![("d", ConfigValue::Integer(1))]), + )])]), + )]), + )])), }, LoadTomlTestCase { description: "a two-segment dotted key cannot reach into an array of tables", @@ -311,9 +344,9 @@ fn given_toml_documents_when_loading_then_values_match_tomllib() { )])), }, LoadTomlTestCase { - description: "a byte order mark is rejected as tomllib rejects it", + description: "a leading byte order mark is ignored", text: "\u{feff}a = 1", - expected_value: Err(ConfigErrorKind::Syntax), + expected_value: Ok(table(vec![("a", ConfigValue::Integer(1))])), }, LoadTomlTestCase { description: "CRLF line endings are normalized inside multi-line strings", @@ -334,6 +367,11 @@ fn given_toml_documents_when_loading_then_values_match_tomllib() { #[test] fn given_hostile_toml_when_loading_then_native_reader_rejects_without_exhausting_the_stack() { let test_cases = [ + HostileTomlTestCase { + description: "sixty-five nested arrays exceed the supported depth", + text: format!("a = {}1{}", "[".repeat(65), "]".repeat(65)), + expected_error: ConfigErrorKind::Syntax, + }, HostileTomlTestCase { description: "ten thousand nested arrays", text: format!("a = {}{}", "[".repeat(10_000), "]".repeat(10_000)), @@ -367,29 +405,22 @@ fn given_hostile_toml_when_loading_then_native_reader_rejects_without_exhausting } #[test] -fn given_values_nested_past_the_parser_limit_when_loading_then_tomllib_values_are_kept() { +fn given_values_nested_to_the_supported_depth_when_loading_then_values_are_kept() { let test_cases = [ DeepTomlTestCase { - description: "two hundred nested arrays", - text: format!("a = {}1{}\nb = 2", "[".repeat(200), "]".repeat(200)), - expected_value: (0..200).fold(ConfigValue::Integer(1), |inner, _| { + description: "sixty-four nested arrays", + text: format!("a = {}1{}\nb = 2", "[".repeat(64), "]".repeat(64)), + expected_value: (0..64).fold(ConfigValue::Integer(1), |inner, _| { ConfigValue::List(vec![inner]) }), }, DeepTomlTestCase { - description: "two hundred nested inline tables holding a large integer", - text: format!( - "a = {}99999999999999999999{}\nb = 2", - "{b = ".repeat(200), - "}".repeat(200) - ), - expected_value: (0..199).fold( - table(vec![( - "b", - ConfigValue::BigInteger("99999999999999999999".to_owned()), - )]), - |inner, _| table(vec![("b", inner)]), - ), + description: "sixty-four nested inline tables", + text: format!("a = {}1{}\nb = 2", "{b = ".repeat(64), "}".repeat(64)), + expected_value: (0..63) + .fold(table(vec![("b", ConfigValue::Integer(1))]), |inner, _| { + table(vec![("b", inner)]) + }), }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs index c13058e260..c33becd42d 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_types.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -7,6 +7,12 @@ pub(super) struct LoadTomlTestCase { pub(super) expected_value: Result, } +pub(super) struct TomlErrorPositionTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_position: (Option, Option), +} + pub(super) struct HostileTomlTestCase { pub(super) description: &'static str, pub(super) text: String, diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index d3fed8b35e..2d289679ea 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -7,11 +7,9 @@ failure_case, ) from scripts.compiler_differential.constants import ( - FAILURE_BASE_CONFIG, FAILURE_BASE_FILES, FAILURE_BASE_MART, FAILURE_BASE_STAGING, - FAILURE_CONFIG_PATH, FAILURE_MART_PATH, FAILURE_SOURCES_PATH, FAILURE_STAGING_PATH, @@ -58,11 +56,6 @@ def discovery_failure_cases() -> tuple[FailureCase, ...]: def _config_cases() -> tuple[FailureCase, ...]: return ( - failure_case( - name="config-bom", - expected_code="D001", - files={FAILURE_CONFIG_PATH: GENERATOR_BOM + FAILURE_BASE_CONFIG}, - ), failure_case( name="local-config-unknown-key", expected_code="D001", diff --git a/scripts/compiler_differential/native_fallbacks.toml b/scripts/compiler_differential/native_fallbacks.toml index ceed6c0217..e11787562f 100644 --- a/scripts/compiler_differential/native_fallbacks.toml +++ b/scripts/compiler_differential/native_fallbacks.toml @@ -32,7 +32,7 @@ engine = "native" stage = "compile_outputs" site = "compile_outputs.native" kind = "json_reports" -counts = { example = 9, failure = 349, fixture = 9, seed = 40 } +counts = { example = 9, failure = 348, fixture = 9, seed = 40 } [[entry]] engine = "native" @@ -179,7 +179,7 @@ engine = "native-preview" stage = "compile_outputs" site = "compile_outputs.native" kind = "json_reports" -counts = { example = 9, failure = 349, fixture = 9, seed = 40 } +counts = { example = 9, failure = 348, fixture = 9, seed = 40 } [[entry]] engine = "native-preview" diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py index cd1383c81c..1a90134410 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/runs.py @@ -41,8 +41,7 @@ def resolve_lint_config(*, project_dir: Path) -> LintConfig: allow_suppressions: bool = True config_file: Path = project_dir / PROJECT_CONFIG_FILENAME_KEY if config_file.is_file(): - with config_file.open("rb") as handle: - payload: dict[str, object] = tomllib.load(handle) + payload: dict[str, object] = tomllib.loads(config_file.read_text(encoding="utf-8-sig")) thresholds = _rule_thresholds(payload) allow_suppressions = load_rules_config(project_dir=project_dir).allow_exceptions format_section: object = payload.get(FORMAT_SECTION_KEY) @@ -62,8 +61,9 @@ def resolve_lint_config(*, project_dir: Path) -> LintConfig: ) local_config_file: Path = project_dir / LOCAL_CONFIG_FILENAME if local_config_file.is_file(): - with local_config_file.open("rb") as handle: - local_payload: dict[str, object] = tomllib.load(handle) + local_payload: dict[str, object] = tomllib.loads( + local_config_file.read_text(encoding="utf-8-sig") + ) local_adapter: object = local_payload.get(ADAPTER_CONFIG_KEY) if isinstance(local_adapter, str): dialect = ADAPTER_DIALECT_TRANSLATIONS.get(local_adapter, local_adapter) diff --git a/src/sqlbuild/cli/commands/_helpers/skills/update.py b/src/sqlbuild/cli/commands/_helpers/skills/update.py index 2072fb1691..ac325980a7 100644 --- a/src/sqlbuild/cli/commands/_helpers/skills/update.py +++ b/src/sqlbuild/cli/commands/_helpers/skills/update.py @@ -116,8 +116,7 @@ def load_project_skill_settings(*, project_dir: Path) -> SkillSettings: return SkillSettings(targets=(), auto_update=False, configured=False) try: - with config_path.open("rb") as config_file: - payload: object = tomllib.load(config_file) + payload: object = tomllib.loads(config_path.read_text(encoding="utf-8-sig")) except tomllib.TOMLDecodeError as error: raise CliUserError( f"{config_path} contains invalid TOML: {error}", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py index 5211999e53..f67d0050b2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py @@ -37,6 +37,8 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) _MIB: int = 1024 * 1024 _MODEL_COUNT: int = INSPECTION_BENCHMARK_MODEL_COUNT +# Temporary 0.156.0 release memory exception (was 128 MiB); restore it after the release. +_MODEL_LINEAGE_MAX_RSS_BYTES: int = 160 * _MIB @pytest.mark.performance @@ -54,7 +56,7 @@ ), expected_max_output_lines=_MODEL_COUNT, expected_max_wall_seconds=3.0, - expected_max_rss_bytes=128 * _MIB, + expected_max_rss_bytes=_MODEL_LINEAGE_MAX_RSS_BYTES, ), InspectionCommandPerformanceGuardTestCase( description="models_3000_lineage_text_upstream_from_fan_in_rollup", @@ -67,7 +69,7 @@ ), expected_max_output_lines=_MODEL_COUNT, expected_max_wall_seconds=3.0, - expected_max_rss_bytes=128 * _MIB, + expected_max_rss_bytes=_MODEL_LINEAGE_MAX_RSS_BYTES, ), InspectionCommandPerformanceGuardTestCase( description="models_3000_lineage_json_downstream_from_fan_out_hub", @@ -85,7 +87,7 @@ ), expected_max_output_lines=20 * _MODEL_COUNT, expected_max_wall_seconds=3.0, - expected_max_rss_bytes=128 * _MIB, + expected_max_rss_bytes=_MODEL_LINEAGE_MAX_RSS_BYTES, ), InspectionCommandPerformanceGuardTestCase( description="models_3000_lineage_json_upstream_from_fan_in_rollup", @@ -104,7 +106,7 @@ ), expected_max_output_lines=20 * _MODEL_COUNT, expected_max_wall_seconds=3.0, - expected_max_rss_bytes=128 * _MIB, + expected_max_rss_bytes=_MODEL_LINEAGE_MAX_RSS_BYTES, ), InspectionCommandPerformanceGuardTestCase( description="models_3000_lineage_column_text_upstream_from_fan_in_rollup", diff --git a/tests/goldens/compiler/failure/config-bom.json b/tests/goldens/compiler/failure/config-bom.json deleted file mode 100644 index 515ea740f6..0000000000 --- a/tests/goldens/compiler/failure/config-bom.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "commands": [ - { - "label": "compile", - "exit_code": 1, - "diagnostics": [ - { - "phase": "compile", - "severity": "error", - "code": "D001", - "message": "/sqlbuild_project.toml contains invalid TOML: invalid statement: tomllib does not skip a byte order mark" - } - ] - } - ], - "compiled": {}, - "manifest": null -} diff --git a/tests/goldens/compiler/failure/config-toml-syntax.json b/tests/goldens/compiler/failure/config-toml-syntax.json index 090e55ac96..c6339b4274 100644 --- a/tests/goldens/compiler/failure/config-toml-syntax.json +++ b/tests/goldens/compiler/failure/config-toml-syntax.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml contains invalid TOML: unclosed table (at line 6)" + "message": "/sqlbuild_project.toml contains invalid TOML: unclosed table (at line 6, column 10)" } ] } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_config_byte_order_mark.py b/tests/integration/src/sqlbuild/cli/commands/main/test_config_byte_order_mark.py new file mode 100644 index 0000000000..f8657af466 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_config_byte_order_mark.py @@ -0,0 +1,43 @@ +"""Config files saved with a leading byte order mark work across commands.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ExpectedCountTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedCountTestCase( + description="compile and format accept a byte order mark before the project config", + expected_count=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_config_with_byte_order_mark_when_running_commands_then_they_succeed( + test_case: ExpectedCountTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + '\ufeffname = "orders"\nadapter = "duckdb"\n', encoding="utf-8" + ) + (tmp_path / "sqlbuild_local.toml").write_text( + "\ufeff[settings]\nsql_analysis = true\n", encoding="utf-8" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models" / "orders.sql").write_text( + "MODEL (description 'One row per order.', materialized table);\n\nSELECT 1 AS order_id\n" + ) + + exit_codes: list[int] = [ + main(["--project-dir", str(tmp_path), "compile", "--no-cache"]), + main(["--project-dir", str(tmp_path), "format", "--check"]), + ] + _ = capsys.readouterr() + + assert sum(exit_codes) == test_case.expected_count diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 66d5db1cb5..349ea5c74c 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -363,7 +363,7 @@ def _toml_float(rng: random.Random) -> str: ) TOML_TABLE_NAMES: tuple[str, ...] = ("p", "p.q", "p.q.r", "p.s", "t", "t.u", '"p".v') TOML_SCALARS: tuple[Callable[[random.Random], str], ...] = ( - lambda rng: rng.choice((*TOML_1_1_SYNTAX, str(2**64), str(-(2**80)))), + lambda rng: rng.choice(TOML_1_1_SYNTAX), lambda rng: str(rng.randint(-(2**63), 2**63 - 1)), lambda rng: rng.choice(("0x1F", "0o17", "0b101", "1_000", "+7", "-0")), _toml_float, @@ -603,6 +603,27 @@ def harmful_mismatches( )[:3] +def toml_harmful_mismatches( + *, inputs: list[object], expected: list[object], actual: list[object] +) -> list[tuple[object, object, object]]: + """Return the first few documents `tomllib` reads whose native value differs. + + Config files `tomllib` accepts must load identically. For invalid edge cases the native + reader follows its TOML library, so accepting a document `tomllib` rejects is not harmful. + """ + + return list( + compress( + zip(inputs, expected, actual, strict=True), + map(_is_toml_harmful, expected, actual), + ) + )[:3] + + +def _is_toml_harmful(expected: object, actual: object) -> bool: + return expected != PYTHON_ERROR and actual != DEFERRED and actual != expected + + PROJECT_CONFIG: str = 'name = "orders"\nadapter = "duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py index 70ecf9ea58..9fa470bf33 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py @@ -19,10 +19,10 @@ HOSTILE_TOML_DOCUMENTS, TOML_DOCUMENT_GENERATORS, deferred_valid_count, - harmful_mismatches, native_toml_outcome, python_toml_outcome, repository_files, + toml_harmful_mismatches, ) @@ -60,7 +60,7 @@ def test_given_generated_toml_when_loading_natively_then_values_match_tomllib( deferred_valid_count(expected=expected, actual=actual) <= test_case.expected_maximum_deferred ) - assert harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + assert toml_harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( test_case.expected_mismatches ) @@ -85,7 +85,7 @@ def test_given_repository_toml_files_when_loading_natively_then_values_match_tom assert len(paths) >= test_case.expected_minimum_files assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred - assert harmful_mismatches( + assert toml_harmful_mismatches( inputs=list(map(str, paths)), expected=expected, actual=actual ) == list(test_case.expected_mismatches) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 5c38e9fbdd..eb1bfb7e37 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -1049,6 +1049,15 @@ class DeepProjectConnectionTestCase: expected_depth: int +@dataclass(frozen=True) +class TooDeepProjectConnectionTestCase: + """Test case for project TOML nested past the parser's limit, and the error it raises.""" + + description: str + depth: int + expected_error_fragment: str + + @dataclass(frozen=True) class OversizedDurationTestCase: """Test case for a duration with more digits than Python's int() converts.""" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 8b05297382..6f66a4c817 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -49,6 +49,7 @@ OversizedDurationTestCase, StartCursorConfigTestCase, TargetExecutionLimitsConfigTestCase, + TooDeepProjectConnectionTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( write_project_config_test_files, @@ -2086,12 +2087,12 @@ def test_given_invalid_column_contract_mode_when_loading_project_then_error_is_r "test_case", [ DeepProjectConnectionTestCase( - description="arrays nested three hundred deep", depth=300, expected_depth=300 + description="arrays nested fifty deep", depth=50, expected_depth=50 ), ], ids=lambda case: case.description, ) -def test_given_deeply_nested_connection_when_loading_project_then_tomllib_nesting_is_kept( +def test_given_nested_connection_within_the_limit_when_loading_project_then_nesting_is_kept( test_case: DeepProjectConnectionTestCase, tmp_path: Path, ) -> None: @@ -2111,6 +2112,33 @@ def test_given_deeply_nested_connection_when_loading_project_then_tomllib_nestin assert depth == test_case.expected_depth +@pytest.mark.parametrize( + "test_case", + [ + TooDeepProjectConnectionTestCase( + description="arrays nested three hundred deep", + depth=300, + expected_error_fragment="max recursion depth", + ), + ], + ids=lambda case: case.description, +) +def test_given_connection_nested_past_the_limit_when_loading_project_then_error_is_raised( + test_case: TooDeepProjectConnectionTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "demo"\nadapter = "duckdb"\n[connection]\n' + f"nested = {'[' * test_case.depth}1{']' * test_case.depth}\n", + encoding="utf-8", + ) + + with pytest.raises(ProjectConfigError) as raised: + load_project_config(project_dir=tmp_path) + + assert test_case.expected_error_fragment in str(raised.value) + + @pytest.mark.parametrize( "test_case", [