From 80c05f630549a7ac00906e603f1d77aae278b5a8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:03:20 +0100 Subject: [PATCH 1/4] feat(native): add core identities, diagnostics, text positions and typed SQL values --- crates/sqlbuild-core/src/lib.rs | 7 +- crates/sqlbuild-core/src/models.rs | 353 ++++++++++++++++++ crates/sqlbuild-core/src/tests.rs | 6 + crates/sqlbuild-core/src/tests/helpers.rs | 64 ++++ crates/sqlbuild-core/src/tests/test_models.rs | 112 ++++++ crates/sqlbuild-core/src/tests/test_types.rs | 26 ++ crates/sqlbuild-core/src/text/errors.rs | 7 + .../src/text/main/decode_python_text.rs | 11 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + crates/sqlbuild-core/src/text/mod.rs | 6 + crates/sqlbuild-core/src/text/models.rs | 101 +++++ crates/sqlbuild-core/src/text/tests.rs | 4 + .../sqlbuild-core/src/text/tests/test_text.rs | 135 +++++++ .../src/text/tests/test_types.rs | 19 + 14 files changed, 851 insertions(+), 1 deletion(-) create mode 100644 crates/sqlbuild-core/src/models.rs create mode 100644 crates/sqlbuild-core/src/tests.rs create mode 100644 crates/sqlbuild-core/src/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/tests/test_models.rs create mode 100644 crates/sqlbuild-core/src/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/text/errors.rs create mode 100644 crates/sqlbuild-core/src/text/main/decode_python_text.rs create mode 100644 crates/sqlbuild-core/src/text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/text/mod.rs create mode 100644 crates/sqlbuild-core/src/text/models.rs create mode 100644 crates/sqlbuild-core/src/text/tests.rs create mode 100644 crates/sqlbuild-core/src/text/tests/test_text.rs create mode 100644 crates/sqlbuild-core/src/text/tests/test_types.rs diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 3912636b36..dc11a3fe2a 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -1,6 +1,11 @@ -//! Shared foundations of the SQLBuild native workspace: constants and the panic boundary. +//! Shared foundations of the SQLBuild native workspace: identities, diagnostics, text and typed values. #![forbid(unsafe_code)] pub mod constants; +pub mod models; pub mod panics; +pub mod text; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/models.rs b/crates/sqlbuild-core/src/models.rs new file mode 100644 index 0000000000..2e5422d295 --- /dev/null +++ b/crates/sqlbuild-core/src/models.rs @@ -0,0 +1,353 @@ +//! Shared compiler identities, diagnostics and typed SQL values mirrored from Python. + +use std::cmp::Ordering; +use std::fmt; + +/// Resource type of a compiled object, Python's `CompiledResourceType`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum CompiledResourceType { + Model, + Source, + Seed, + Udf, + TableFn, + DbtRef, + Audit, + SqlTest, + SqlScenario, +} + +impl CompiledResourceType { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Source => "source", + Self::Seed => "seed", + Self::Udf => "udf", + Self::TableFn => "table_fn", + Self::DbtRef => "dbt_ref", + Self::Audit => "audit", + Self::SqlTest => "sql_test", + Self::SqlScenario => "sql_scenario", + } + } +} + +/// Logical identity of one compiled resource or external dependency, Python's `CompiledObjectKey`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct CompiledObjectKey { + pub resource_type: CompiledResourceType, + pub name: String, +} + +/// Kind of an authored resource in the scope index, Python's `ResourceKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ResourceKind { + Model, + Test, + Scenario, + Function, + Source, + Seed, +} + +impl ResourceKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Test => "test", + Self::Scenario => "scenario", + Self::Function => "function", + Self::Source => "source", + Self::Seed => "seed", + } + } +} + +/// Independent declaration namespace, Python's `DeclarationKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DeclarationKind { + Macro, + Enum, + Constant, + Audit, + SingularAudit, + Schema, + SqlHook, + PythonHook, +} + +impl DeclarationKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Macro => "macro", + Self::Enum => "enum", + Self::Constant => "constant", + Self::Audit => "audit", + Self::SingularAudit => "singular_audit", + Self::Schema => "schema", + Self::SqlHook => "sql_hook", + Self::PythonHook => "python_hook", + } + } +} + +/// Kind-qualified identity of one authored resource, displayed as `kind:name`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ResourceIdentity { + pub kind: ResourceKind, + pub name: String, +} + +/// Declaration identity, displayed as `kind:name` or `kind:owner_kind:owner_name.name`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct DeclarationIdentity { + pub kind: DeclarationKind, + pub name: String, + pub owner: Option, +} + +/// Orders by kind value then name, as Python's `order=True` dataclass compares its fields. +impl Ord for ResourceIdentity { + fn cmp(&self, other: &Self) -> Ordering { + (self.kind.as_str(), self.name.as_str()).cmp(&(other.kind.as_str(), other.name.as_str())) + } +} + +impl PartialOrd for ResourceIdentity { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Orders by kind value, name, then owner; a public declaration sorts before a private twin. +impl Ord for DeclarationIdentity { + fn cmp(&self, other: &Self) -> Ordering { + (self.kind.as_str(), self.name.as_str(), &self.owner).cmp(&( + other.kind.as_str(), + other.name.as_str(), + &other.owner, + )) + } +} + +impl PartialOrd for DeclarationIdentity { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// The text of Python's `format_identity`. +impl fmt::Display for ResourceIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.kind.as_str(), self.name) + } +} + +/// The text of Python's `format_identity`. +impl fmt::Display for DeclarationIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.owner { + None => write!(formatter, "{}:{}", self.kind.as_str(), self.name), + Some(owner) => write!( + formatter, + "{}:{}:{}.{}", + self.kind.as_str(), + owner.kind.as_str(), + owner.name, + self.name + ), + } + } +} + +/// Phase that produced a diagnostic, Python's `DiagnosticPhase`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DiagnosticPhase { + Compile, + Contract, + Rule, + Plan, + Build, + Audit, + Test, + Connection, +} + +impl DiagnosticPhase { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Compile => "compile", + Self::Contract => "contract", + Self::Rule => "rule", + Self::Plan => "plan", + Self::Build => "build", + Self::Audit => "audit", + Self::Test => "test", + Self::Connection => "connection", + } + } +} + +/// Severity of a diagnostic, Python's `DiagnosticSeverity`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DiagnosticSeverity { + Error, + Warning, + Info, +} + +impl DiagnosticSeverity { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Error => "error", + Self::Warning => "warning", + Self::Info => "info", + } + } +} + +/// An authored location with 1-based lines and code-point columns, Python's `SourceLocation`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SourceLocation { + pub path: String, + pub line: usize, + pub column: usize, + pub end_line: Option, + pub end_column: Option, +} + +/// A secondary location that adds context to a diagnostic, Python's `RelatedLocation`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RelatedLocation { + pub label: String, + pub location: SourceLocation, + pub message: Option, +} + +/// Serial emission position: resource compile order, then stage, then sequence within it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DiagnosticOrderKey { + pub resource_order: u32, + pub stage: u32, + pub sequence: u32, +} + +/// One compile-time diagnostic, Python's `CompilerDiagnostic` plus its serial order key. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Diagnostic { + pub phase: DiagnosticPhase, + pub severity: DiagnosticSeverity, + pub code: String, + pub message: String, + pub resource_type: Option, + pub resource_name: Option, + pub column_name: Option, + pub location: Option, + pub related_locations: Vec, + pub help: Option, + pub notes: Vec, + pub affected_rules: Vec, + pub order_key: DiagnosticOrderKey, +} + +impl Diagnostic { + /// Whether this diagnostic fails the command. + pub fn is_error(&self) -> bool { + self.severity == DiagnosticSeverity::Error + } +} + +/// Closed logical kinds of typed SQL values, Python's `SqlValueKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum SqlValueKind { + String, + Integer, + Boolean, + Float, + Decimal, + Null, + List, + Set, + Object, +} + +impl SqlValueKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::String => "string", + Self::Integer => "integer", + Self::Boolean => "boolean", + Self::Float => "float", + Self::Decimal => "decimal", + Self::Null => "null", + Self::List => "list", + Self::Set => "set", + Self::Object => "object", + } + } +} + +/// Logical type of a typed SQL value; collections carry their element type. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SqlLogicalType { + pub kind: SqlValueKind, + pub element_type: Option>, +} + +impl SqlLogicalType { + /// Python's `display_name`, such as `list`. + pub fn display_name(&self) -> String { + match &self.element_type { + None => self.kind.as_str().to_owned(), + Some(element) => format!("{}<{}>", self.kind.as_str(), element.display_name()), + } + } +} + +/// One validated, canonical typed SQL value, Python's normalized `SqlValue` (see crates/README.md). +#[derive(Clone, Debug, PartialEq)] +pub enum SqlValue { + Null, + Boolean(bool), + Integer(i64), + Float(f64), + Decimal(String), + String(String), + List { + element_type: SqlLogicalType, + values: Vec, + }, + Set { + element_type: SqlLogicalType, + values: Vec, + }, + Object(Vec<(String, SqlValue)>), +} + +impl SqlValue { + /// The logical type of this value. + pub fn logical_type(&self) -> SqlLogicalType { + let (kind, element_type) = match self { + Self::Null => (SqlValueKind::Null, None), + Self::Boolean(_) => (SqlValueKind::Boolean, None), + Self::Integer(_) => (SqlValueKind::Integer, None), + Self::Float(_) => (SqlValueKind::Float, None), + Self::Decimal(_) => (SqlValueKind::Decimal, None), + Self::String(_) => (SqlValueKind::String, None), + Self::List { element_type, .. } => (SqlValueKind::List, Some(element_type)), + Self::Set { element_type, .. } => (SqlValueKind::Set, Some(element_type)), + Self::Object(_) => (SqlValueKind::Object, None), + }; + SqlLogicalType { + kind, + element_type: element_type.map(|element| Box::new(element.clone())), + } + } +} diff --git a/crates/sqlbuild-core/src/tests.rs b/crates/sqlbuild-core/src/tests.rs new file mode 100644 index 0000000000..2ad132af82 --- /dev/null +++ b/crates/sqlbuild-core/src/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_models.rs"] +mod models; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/tests/helpers.rs b/crates/sqlbuild-core/src/tests/helpers.rs new file mode 100644 index 0000000000..a072240428 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/helpers.rs @@ -0,0 +1,64 @@ +use crate::models::{ + DeclarationIdentity, DeclarationKind, Diagnostic, DiagnosticOrderKey, DiagnosticPhase, + DiagnosticSeverity, ResourceIdentity, ResourceKind, SqlLogicalType, SqlValue, SqlValueKind, +}; + +pub(super) fn declaration(kind: DeclarationKind, name: &str) -> DeclarationIdentity { + DeclarationIdentity { + kind, + name: name.to_owned(), + owner: None, + } +} + +pub(super) fn private_constant(owner: &str, name: &str) -> DeclarationIdentity { + DeclarationIdentity { + owner: Some(ResourceIdentity { + kind: ResourceKind::Model, + name: owner.to_owned(), + }), + ..declaration(DeclarationKind::Constant, name) + } +} + +pub(super) fn diagnostic(code: &str, severity: DiagnosticSeverity, order: [u32; 3]) -> Diagnostic { + Diagnostic { + phase: DiagnosticPhase::Compile, + severity, + code: code.to_owned(), + message: format!("{code} message"), + resource_type: None, + resource_name: None, + column_name: None, + location: None, + related_locations: vec![], + help: None, + notes: vec![], + affected_rules: vec![], + order_key: DiagnosticOrderKey { + resource_order: order[0], + stage: order[1], + sequence: order[2], + }, + } +} + +pub(super) fn scalar_type(kind: SqlValueKind) -> SqlLogicalType { + SqlLogicalType { + kind, + element_type: None, + } +} + +pub(super) fn list_of_integer_sets() -> SqlValue { + SqlValue::List { + element_type: SqlLogicalType { + kind: SqlValueKind::Set, + element_type: Some(Box::new(scalar_type(SqlValueKind::Integer))), + }, + values: vec![SqlValue::Set { + element_type: scalar_type(SqlValueKind::Integer), + values: vec![SqlValue::Integer(1), SqlValue::Integer(2)], + }], + } +} diff --git a/crates/sqlbuild-core/src/tests/test_models.rs b/crates/sqlbuild-core/src/tests/test_models.rs new file mode 100644 index 0000000000..f12249ecc2 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/test_models.rs @@ -0,0 +1,112 @@ +use crate::models::{DeclarationKind, Diagnostic, DiagnosticSeverity, SqlValue}; +use crate::tests::helpers::{declaration, diagnostic, list_of_integer_sets, private_constant}; +use crate::tests::test_types::{ + DiagnosticOrderTestCase, IdentityOrderTestCase, IdentityTextTestCase, SqlTypeNameTestCase, +}; + +#[test] +fn given_identities_when_formatting_then_text_matches_python_format_identity() { + let test_cases = [ + IdentityTextTestCase { + description: "public declaration", + identity: declaration(DeclarationKind::SqlHook, "grant_reader"), + expected_text: "sql_hook:grant_reader", + }, + IdentityTextTestCase { + description: "private declaration names its owner", + identity: private_constant("staging.orders", "status"), + expected_text: "constant:model:staging.orders.status", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.identity.to_string(), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_identities_when_sorting_then_order_follows_python_enum_values() { + let test_cases = [IdentityOrderTestCase { + description: "kinds compare by their string values, then names, then owners", + identities: vec![ + declaration(DeclarationKind::Schema, "a"), + private_constant("orders", "b"), + declaration(DeclarationKind::Macro, "a"), + declaration(DeclarationKind::Constant, "b"), + declaration(DeclarationKind::Audit, "z"), + ], + expected_texts: vec![ + "audit:z", + "constant:b", + "constant:model:orders.b", + "macro:a", + "schema:a", + ], + }]; + for test_case in test_cases { + let mut identities = test_case.identities; + identities.sort(); + let texts: Vec = identities.iter().map(ToString::to_string).collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } +} + +#[test] +fn given_diagnostics_from_parallel_work_when_sorting_by_order_key_then_serial_order_returns() { + let test_cases = [DiagnosticOrderTestCase { + description: "resource order, then stage, then sequence", + diagnostics: vec![ + diagnostic("C003", DiagnosticSeverity::Warning, [1, 0, 0]), + diagnostic("C002", DiagnosticSeverity::Error, [0, 1, 0]), + diagnostic("C001", DiagnosticSeverity::Info, [0, 0, 1]), + diagnostic("C000", DiagnosticSeverity::Error, [0, 0, 0]), + ], + expected_codes: vec!["C000", "C001", "C002", "C003"], + expected_errors: vec![true, false, true, false], + }]; + for test_case in test_cases { + let mut diagnostics = test_case.diagnostics; + diagnostics.sort_by_key(|item| item.order_key); + let codes: Vec<&str> = diagnostics.iter().map(|item| item.code.as_str()).collect(); + let errors: Vec = diagnostics.iter().map(Diagnostic::is_error).collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + assert_eq!( + errors, test_case.expected_errors, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_typed_values_when_naming_types_then_names_match_python_display_name() { + let test_cases = [ + SqlTypeNameTestCase { + description: "nested collections name every element type", + value: list_of_integer_sets(), + expected_name: "list>", + }, + SqlTypeNameTestCase { + description: "scalar", + value: SqlValue::Float(1.5), + expected_name: "float", + }, + SqlTypeNameTestCase { + description: "object has no element type", + value: SqlValue::Object(vec![("status".to_owned(), SqlValue::Null)]), + expected_name: "object", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.value.logical_type().display_name(), + test_case.expected_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/tests/test_types.rs b/crates/sqlbuild-core/src/tests/test_types.rs new file mode 100644 index 0000000000..7550859b12 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/test_types.rs @@ -0,0 +1,26 @@ +use crate::models::{DeclarationIdentity, Diagnostic, SqlValue}; + +pub(super) struct IdentityTextTestCase { + pub(super) description: &'static str, + pub(super) identity: DeclarationIdentity, + pub(super) expected_text: &'static str, +} + +pub(super) struct IdentityOrderTestCase { + pub(super) description: &'static str, + pub(super) identities: Vec, + pub(super) expected_texts: Vec<&'static str>, +} + +pub(super) struct DiagnosticOrderTestCase { + pub(super) description: &'static str, + pub(super) diagnostics: Vec, + pub(super) expected_codes: Vec<&'static str>, + pub(super) expected_errors: Vec, +} + +pub(super) struct SqlTypeNameTestCase { + pub(super) description: &'static str, + pub(super) value: SqlValue, + pub(super) expected_name: &'static str, +} diff --git a/crates/sqlbuild-core/src/text/errors.rs b/crates/sqlbuild-core/src/text/errors.rs new file mode 100644 index 0000000000..d69651f2cf --- /dev/null +++ b/crates/sqlbuild-core/src/text/errors.rs @@ -0,0 +1,7 @@ +//! Errors of decoding authored text. + +/// Authored bytes that are not valid UTF-8; `valid_up_to` is the longest valid prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TextDecodeError { + pub valid_up_to: usize, +} diff --git a/crates/sqlbuild-core/src/text/main/decode_python_text.rs b/crates/sqlbuild-core/src/text/main/decode_python_text.rs new file mode 100644 index 0000000000..386c039cff --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/decode_python_text.rs @@ -0,0 +1,11 @@ +//! Decode authored file bytes the way Python's `Path.read_text(encoding="utf-8")` does. + +use crate::text::errors::TextDecodeError; + +/// Decode strict UTF-8 with universal newlines (`\r\n` and `\r` become `\n`), keeping a BOM. +pub fn decode_python_text(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).map_err(|error| TextDecodeError { + valid_up_to: error.valid_up_to(), + })?; + Ok(text.replace("\r\n", "\n").replace('\r', "\n")) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs new file mode 100644 index 0000000000..57e2d7ac2e --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -0,0 +1 @@ +pub mod decode_python_text; diff --git a/crates/sqlbuild-core/src/text/mod.rs b/crates/sqlbuild-core/src/text/mod.rs new file mode 100644 index 0000000000..ed469c77cd --- /dev/null +++ b/crates/sqlbuild-core/src/text/mod.rs @@ -0,0 +1,6 @@ +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/text/models.rs b/crates/sqlbuild-core/src/text/models.rs new file mode 100644 index 0000000000..7ba97cb723 --- /dev/null +++ b/crates/sqlbuild-core/src/text/models.rs @@ -0,0 +1,101 @@ +//! Positions in authored text, in both UTF-8 bytes and Python code points. + +/// One text position; lines and columns are 1-based and columns count code points. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct TextPosition { + pub byte_offset: usize, + pub char_offset: usize, + pub line: usize, + pub column: usize, +} + +/// A half-open range of text. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TextSpan { + pub start: TextPosition, + pub end: TextPosition, +} + +/// Line starts of one text, converting offsets into Python-compatible lines and columns. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineIndex<'text> { + text: &'text str, + line_byte_starts: Vec, + line_char_starts: Vec, + char_len: usize, +} + +impl<'text> LineIndex<'text> { + /// Index the line starts of `text`; only `\n` ends a line, as in Python's `str.count`. + pub fn new(text: &'text str) -> Self { + let mut line_byte_starts = vec![0]; + let mut line_char_starts = vec![0]; + let mut char_len = 0; + for (byte_offset, character) in text.char_indices() { + char_len += 1; + if character == '\n' { + line_byte_starts.push(byte_offset + 1); + line_char_starts.push(char_len); + } + } + Self { + text, + line_byte_starts, + line_char_starts, + char_len, + } + } + + /// The position at a UTF-8 byte offset, or `None` past the end or inside a character. + pub fn position_at_byte(&self, byte_offset: usize) -> Option { + if byte_offset > self.text.len() || !self.text.is_char_boundary(byte_offset) { + return None; + } + let line_index = self + .line_byte_starts + .partition_point(|start| *start <= byte_offset) + - 1; + let line_start = self.line_byte_starts[line_index]; + let column = self.text[line_start..byte_offset].chars().count() + 1; + Some(TextPosition { + byte_offset, + char_offset: self.line_char_starts[line_index] + column - 1, + line: line_index + 1, + column, + }) + } + + /// The position at a code-point offset (a Python string index), or `None` past the end. + pub fn position_at_char(&self, char_offset: usize) -> Option { + if char_offset > self.char_len { + return None; + } + let line_index = self + .line_char_starts + .partition_point(|start| *start <= char_offset) + - 1; + let line_start = self.line_byte_starts[line_index]; + let column = char_offset - self.line_char_starts[line_index] + 1; + let byte_offset = self.text[line_start..] + .char_indices() + .nth(column - 1) + .map_or(self.text.len(), |(offset, _)| line_start + offset); + Some(TextPosition { + byte_offset, + char_offset, + line: line_index + 1, + column, + }) + } + + /// The span between two UTF-8 byte offsets, or `None` when either end is not a position. + pub fn span_at_bytes(&self, start: usize, end: usize) -> Option { + if start > end { + return None; + } + Some(TextSpan { + start: self.position_at_byte(start)?, + end: self.position_at_byte(end)?, + }) + } +} diff --git a/crates/sqlbuild-core/src/text/tests.rs b/crates/sqlbuild-core/src/text/tests.rs new file mode 100644 index 0000000000..26f62e1540 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_text.rs"] +mod text; diff --git a/crates/sqlbuild-core/src/text/tests/test_text.rs b/crates/sqlbuild-core/src/text/tests/test_text.rs new file mode 100644 index 0000000000..a59c4164c4 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_text.rs @@ -0,0 +1,135 @@ +use crate::text::main::decode_python_text::decode_python_text; +use crate::text::models::LineIndex; +use crate::text::tests::test_types::{DecodeTestCase, PositionTestCase, SpanTestCase}; + +#[test] +fn given_offsets_when_indexing_text_then_lines_and_columns_match_python() { + let test_cases = [ + PositionTestCase { + description: "start of text", + text: "select 1", + byte_offset: 0, + expected_position: Some((0, 1, 1)), + }, + PositionTestCase { + description: "end of text is addressable", + text: "select 1", + byte_offset: 8, + expected_position: Some((8, 1, 9)), + }, + PositionTestCase { + description: "multi-byte characters count once", + text: "-- café ☕ 😀\nselect é", + byte_offset: 25, + expected_position: Some((19, 2, 8)), + }, + PositionTestCase { + description: "tab counts as one column", + text: "\tselect\t1", + byte_offset: 8, + expected_position: Some((8, 1, 9)), + }, + PositionTestCase { + description: "newline belongs to the line it ends", + text: "a\nb", + byte_offset: 1, + expected_position: Some((1, 1, 2)), + }, + PositionTestCase { + description: "carriage return is an ordinary column", + text: "a\r\nb", + byte_offset: 3, + expected_position: Some((3, 2, 1)), + }, + PositionTestCase { + description: "offset inside a character is rejected", + text: "é", + byte_offset: 1, + expected_position: None, + }, + PositionTestCase { + description: "offset past the end is rejected", + text: "a", + byte_offset: 2, + expected_position: None, + }, + ]; + for test_case in test_cases { + let index = LineIndex::new(test_case.text); + let position = index.position_at_byte(test_case.byte_offset); + let round_trip = position.and_then(|found| index.position_at_char(found.char_offset)); + assert_eq!( + position.map(|found| (found.char_offset, found.line, found.column)), + test_case.expected_position, + "{}", + test_case.description + ); + assert_eq!(round_trip, position, "{} round trip", test_case.description); + } +} + +#[test] +fn given_byte_ranges_when_building_spans_then_both_ends_are_positions() { + let test_cases = [ + SpanTestCase { + description: "range on the second line", + text: "one\ntwo é three", + byte_range: (8, 10), + expected_ends: Some(((2, 5), (2, 6))), + }, + SpanTestCase { + description: "reversed range is rejected", + text: "one", + byte_range: (2, 1), + expected_ends: None, + }, + ]; + for test_case in test_cases { + let span = LineIndex::new(test_case.text) + .span_at_bytes(test_case.byte_range.0, test_case.byte_range.1); + assert_eq!( + span.map(|found| ( + (found.start.line, found.start.column), + (found.end.line, found.end.column) + )), + test_case.expected_ends, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_authored_bytes_when_decoding_then_text_matches_python_read_text() { + let test_cases = [ + DecodeTestCase { + description: "CRLF and lone CR become LF", + bytes: b"a\r\nb\rc\n\r\n", + expected_text: Ok("a\nb\nc\n\n"), + }, + DecodeTestCase { + description: "byte-order mark is kept", + bytes: b"\xef\xbb\xbfselect", + expected_text: Ok("\u{feff}select"), + }, + DecodeTestCase { + description: "encoded surrogate is invalid", + bytes: b"ok\xed\xa0\x80", + expected_text: Err(2), + }, + DecodeTestCase { + description: "truncated sequence is invalid", + bytes: b"caf\xc3", + expected_text: Err(3), + }, + ]; + for test_case in test_cases { + let actual = decode_python_text(test_case.bytes).map_err(|error| error.valid_up_to); + assert_eq!( + actual.as_deref().map_err(|offset| *offset), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs new file mode 100644 index 0000000000..850689e5e1 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -0,0 +1,19 @@ +pub(super) struct PositionTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) byte_offset: usize, + pub(super) expected_position: Option<(usize, usize, usize)>, +} + +pub(super) struct SpanTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) byte_range: (usize, usize), + pub(super) expected_ends: Option<((usize, usize), (usize, usize))>, +} + +pub(super) struct DecodeTestCase { + pub(super) description: &'static str, + pub(super) bytes: &'static [u8], + pub(super) expected_text: Result<&'static str, usize>, +} From 2134f4ff8aa345fa4ff334c9cb1cff3c039f7501 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:16:17 +0100 Subject: [PATCH 2/4] feat(native): add an exact Python JSON emitter with oracle tests The emitter refuses integers beyond Python's 4300-digit string limit and values nested deeper than 128 levels instead of exhausting the stack. orjson is required at 3.11.7 or later, whose float layout the emitter reproduces, and a test pins that layout. --- .../src/json/_helpers/emitter.rs | 143 +++++++++ .../sqlbuild-core/src/json/_helpers/floats.rs | 74 +++++ crates/sqlbuild-core/src/json/_helpers/mod.rs | 3 + .../src/json/_helpers/strings.rs | 50 +++ crates/sqlbuild-core/src/json/constants.rs | 13 + crates/sqlbuild-core/src/json/errors.rs | 14 + crates/sqlbuild-core/src/json/main/dumps.rs | 9 + crates/sqlbuild-core/src/json/main/mod.rs | 1 + crates/sqlbuild-core/src/json/mod.rs | 8 + crates/sqlbuild-core/src/json/models.rs | 138 ++++++++ crates/sqlbuild-core/src/json/tests.rs | 6 + .../sqlbuild-core/src/json/tests/helpers.rs | 57 ++++ .../src/json/tests/test_dumps.rs | 267 ++++++++++++++++ .../src/json/tests/test_types.rs | 21 ++ crates/sqlbuild-core/src/lib.rs | 3 +- .../src/bindings/_helpers/functions.rs | 1 + .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/oracles.rs | 137 ++++++++ pyproject.toml | 2 +- src/sqlbuild/_native.pyi | 6 + .../src/sqlbuild/compiler/_test_types.py | 39 +++ .../src/sqlbuild/compiler/helpers.py | 301 ++++++++++++++++++ .../compiler/test_native_json_oracle.py | 149 +++++++++ .../compiler/test_native_text_oracle.py | 86 +++++ uv.lock | 2 +- 25 files changed, 1528 insertions(+), 3 deletions(-) create mode 100644 crates/sqlbuild-core/src/json/_helpers/emitter.rs create mode 100644 crates/sqlbuild-core/src/json/_helpers/floats.rs create mode 100644 crates/sqlbuild-core/src/json/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/json/_helpers/strings.rs create mode 100644 crates/sqlbuild-core/src/json/constants.rs create mode 100644 crates/sqlbuild-core/src/json/errors.rs create mode 100644 crates/sqlbuild-core/src/json/main/dumps.rs create mode 100644 crates/sqlbuild-core/src/json/main/mod.rs create mode 100644 crates/sqlbuild-core/src/json/mod.rs create mode 100644 crates/sqlbuild-core/src/json/models.rs create mode 100644 crates/sqlbuild-core/src/json/tests.rs create mode 100644 crates/sqlbuild-core/src/json/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/json/tests/test_dumps.rs create mode 100644 crates/sqlbuild-core/src/json/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/oracles.rs create mode 100644 tests/integration/src/sqlbuild/compiler/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py create mode 100644 tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py diff --git a/crates/sqlbuild-core/src/json/_helpers/emitter.rs b/crates/sqlbuild-core/src/json/_helpers/emitter.rs new file mode 100644 index 0000000000..809c66b1d5 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/emitter.rs @@ -0,0 +1,143 @@ +//! Serialize a [`JsonValue`] in the layout of one Python serializer. + +use crate::json::_helpers::floats::{orjson_text, python_repr}; +use crate::json::_helpers::strings::{StringEscaping, json_string}; +use crate::json::constants::{ + MAX_NESTING_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, +}; +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue}; + +/// The resolved layout of one serializer configuration. +struct Layout<'options> { + indent: Option<&'options str>, + item_separator: &'options str, + key_separator: &'options str, + sort_keys: bool, + escaping: StringEscaping, + orjson: bool, + allow_nan: bool, +} + +impl<'options> Layout<'options> { + fn new(dialect: &'options JsonDialect) -> Self { + match dialect { + JsonDialect::Stdlib(options) => Self { + indent: options.indent.as_deref(), + item_separator: &options.item_separator, + key_separator: &options.key_separator, + sort_keys: options.sort_keys, + escaping: if options.ensure_ascii { + StringEscaping::Ascii + } else { + StringEscaping::Unicode + }, + orjson: false, + allow_nan: options.allow_nan, + }, + JsonDialect::Orjson(options) => Self { + indent: options.indent_2.then_some(" "), + item_separator: ",", + key_separator: if options.indent_2 { ": " } else { ":" }, + sort_keys: options.sort_keys, + escaping: StringEscaping::Unicode, + orjson: true, + allow_nan: true, + }, + } + } + + fn newline(&self, level: usize) -> String { + self.indent + .map_or_else(String::new, |indent| format!("\n{}", indent.repeat(level))) + } + + fn integer(&self, value: &JsonInteger) -> Result { + let in_range = value + .to_i128() + .is_some_and(|number| (ORJSON_MIN_INTEGER..=ORJSON_MAX_INTEGER).contains(&number)); + if self.orjson && !in_range { + return Err(JsonEmitError::IntegerOutOfRange); + } + let digits = value.as_str().trim_start_matches('-').len(); + if digits > PYTHON_INT_MAX_STR_DIGITS { + return Err(JsonEmitError::IntegerTooLong); + } + Ok(value.as_str().to_owned()) + } + + fn float(&self, value: f64) -> Result { + match value { + _ if value.is_finite() && self.orjson => Ok(orjson_text(value)), + _ if value.is_finite() => Ok(python_repr(value)), + _ if self.orjson => Ok("null".to_owned()), + _ if !self.allow_nan => Err(JsonEmitError::NonFiniteFloat), + _ if value.is_nan() => Ok("NaN".to_owned()), + _ if value.is_sign_positive() => Ok("Infinity".to_owned()), + _ => Ok("-Infinity".to_owned()), + } + } + + fn value(&self, value: &JsonValue, level: usize) -> Result { + if level > MAX_NESTING_DEPTH { + return Err(JsonEmitError::NestingTooDeep); + } + match value { + JsonValue::Null => Ok("null".to_owned()), + JsonValue::Bool(flag) => Ok(flag.to_string()), + JsonValue::Integer(number) => self.integer(number), + JsonValue::Float(number) => self.float(*number), + JsonValue::String(text) => Ok(json_string(text, self.escaping)), + JsonValue::Array(items) => { + let members = items + .iter() + .map(|item| self.value(item, level + 1)) + .collect::, _>>()?; + Ok(self.container(('[', ']'), members, level)) + } + JsonValue::Object(entries) => self.object(entries, level), + } + } + + fn object( + &self, + entries: &[(String, JsonValue)], + level: usize, + ) -> Result { + let mut ordered: Vec<&(String, JsonValue)> = entries.iter().collect(); + if self.sort_keys { + ordered.sort_by(|left, right| left.0.cmp(&right.0)); + } + let members = ordered + .into_iter() + .map(|(key, item)| { + Ok(format!( + "{}{}{}", + json_string(key, self.escaping), + self.key_separator, + self.value(item, level + 1)? + )) + }) + .collect::, _>>()?; + Ok(self.container(('{', '}'), members, level)) + } + + fn container(&self, brackets: (char, char), members: Vec, level: usize) -> String { + if members.is_empty() { + return format!("{}{}", brackets.0, brackets.1); + } + let opening = self.newline(level + 1); + let separator = format!("{}{opening}", self.item_separator); + let closing = self.newline(level); + format!( + "{}{opening}{}{closing}{}", + brackets.0, + members.join(&separator), + brackets.1 + ) + } +} + +pub(crate) fn emit(value: &JsonValue, dialect: &JsonDialect) -> Result { + Layout::new(dialect).value(value, 0) +} diff --git a/crates/sqlbuild-core/src/json/_helpers/floats.rs b/crates/sqlbuild-core/src/json/_helpers/floats.rs new file mode 100644 index 0000000000..04ebd950a1 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/floats.rs @@ -0,0 +1,74 @@ +//! Shortest round-trip float text in Python's and orjson's layouts. + +use crate::json::constants::{ + ORJSON_EXPONENT_DIGITS, ORJSON_FIXED_EXPONENTS, PYTHON_EXPONENT_DIGITS, PYTHON_FIXED_EXPONENTS, +}; +use std::ops::RangeInclusive; + +fn scientific_parts(text: &str) -> (String, i32) { + let (mantissa, exponent) = text.split_once('e').unwrap_or((text, "0")); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + let (sign, magnitude) = exponent + .strip_prefix('-') + .map_or((1, exponent), |magnitude| (-1, magnitude)); + let exponent = magnitude + .bytes() + .fold(0_i32, |total, digit| total * 10 + i32::from(digit - b'0')); + (digits, sign * exponent) +} + +/// Shortest round-trip digits of `|value|`, ties rounded half to even as CPython does. +fn shortest_digits(value: f64) -> (String, i32) { + let (shortest, _) = scientific_parts(&format!("{:e}", value.abs())); + let precision = shortest.len().saturating_sub(1); + scientific_parts(&format!("{:.precision$e}", value.abs())) +} + +/// Fixed-point layout of `digits` with the decimal point after `point` digits. +fn fixed(digits: &str, point: i32) -> String { + match usize::try_from(point) { + Err(_) | Ok(0) => format!("0.{}{digits}", "0".repeat(point.unsigned_abs() as usize)), + Ok(point) if point >= digits.len() => { + format!("{digits}{}.0", "0".repeat(point - digits.len())) + } + Ok(point) => format!("{}.{}", &digits[..point], &digits[point..]), + } +} + +fn scientific(digits: &str, exponent: i32, exponent_digits: usize) -> String { + let (first, rest) = digits.split_at(1); + let mantissa = if rest.is_empty() { + first.to_owned() + } else { + format!("{first}.{rest}") + }; + let sign = if exponent.is_negative() { '-' } else { '+' }; + let magnitude = exponent.unsigned_abs(); + format!("{mantissa}e{sign}{magnitude:0exponent_digits$}") +} + +fn layout(value: f64, fixed_exponents: RangeInclusive, exponent_digits: usize) -> String { + let (digits, exponent) = shortest_digits(value); + let body = if fixed_exponents.contains(&exponent) { + fixed(&digits, exponent + 1) + } else { + scientific(&digits, exponent, exponent_digits) + }; + let sign = if value.is_sign_negative() { "-" } else { "" }; + format!("{sign}{body}") +} + +/// Python's `repr(float)`, including `nan`, `inf` and `-inf`. +pub(crate) fn python_repr(value: f64) -> String { + match value { + _ if value.is_nan() => "nan".to_owned(), + _ if value.is_infinite() && value.is_sign_positive() => "inf".to_owned(), + _ if value.is_infinite() => "-inf".to_owned(), + _ => layout(value, PYTHON_FIXED_EXPONENTS, PYTHON_EXPONENT_DIGITS), + } +} + +/// orjson's text for a finite float. +pub(crate) fn orjson_text(value: f64) -> String { + layout(value, ORJSON_FIXED_EXPONENTS, ORJSON_EXPONENT_DIGITS) +} diff --git a/crates/sqlbuild-core/src/json/_helpers/mod.rs b/crates/sqlbuild-core/src/json/_helpers/mod.rs new file mode 100644 index 0000000000..44402147f5 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod emitter; +pub(crate) mod floats; +pub(crate) mod strings; diff --git a/crates/sqlbuild-core/src/json/_helpers/strings.rs b/crates/sqlbuild-core/src/json/_helpers/strings.rs new file mode 100644 index 0000000000..169c27da2e --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/strings.rs @@ -0,0 +1,50 @@ +//! JSON string escaping of `json.dumps` and orjson. + +use crate::json::constants::FIRST_PRINTABLE_ASCII; + +/// How a serializer escapes characters outside printable ASCII. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StringEscaping { + /// `ensure_ascii=True`: `\uXXXX`, with UTF-16 surrogate pairs above U+FFFF. + Ascii, + /// `ensure_ascii=False` and orjson: only control characters are escaped. + Unicode, +} + +fn unicode_escape(unit: u32) -> String { + format!("\\u{unit:04x}") +} + +fn escaped(character: char, escaping: StringEscaping) -> String { + match character { + '"' => "\\\"".to_owned(), + '\\' => "\\\\".to_owned(), + '\n' => "\\n".to_owned(), + '\r' => "\\r".to_owned(), + '\t' => "\\t".to_owned(), + '\u{8}' => "\\b".to_owned(), + '\u{c}' => "\\f".to_owned(), + ' '..='~' => character.to_string(), + _ if character < FIRST_PRINTABLE_ASCII => unicode_escape(u32::from(character)), + _ if escaping == StringEscaping::Unicode => character.to_string(), + _ => character + .encode_utf16(&mut [0_u16; 2]) + .iter() + .map(|unit| unicode_escape(u32::from(*unit))) + .collect(), + } +} + +/// `text` as a quoted JSON string. +pub(crate) fn json_string(text: &str, escaping: StringEscaping) -> String { + let mut output = String::with_capacity(text.len() + 2); + output.push('"'); + for character in text.chars() { + match character { + ' '..='~' if character != '"' && character != '\\' => output.push(character), + _ => output.push_str(&escaped(character, escaping)), + } + } + output.push('"'); + output +} diff --git a/crates/sqlbuild-core/src/json/constants.rs b/crates/sqlbuild-core/src/json/constants.rs new file mode 100644 index 0000000000..fab04a225b --- /dev/null +++ b/crates/sqlbuild-core/src/json/constants.rs @@ -0,0 +1,13 @@ +//! Serializer constants. + +pub(crate) const ORJSON_MIN_INTEGER: i128 = i64::MIN as i128; +pub(crate) const ORJSON_MAX_INTEGER: i128 = u64::MAX as i128; +pub(crate) const FIRST_PRINTABLE_ASCII: char = ' '; +pub(crate) const PYTHON_FIXED_EXPONENTS: std::ops::RangeInclusive = -4..=15; +pub(crate) const ORJSON_FIXED_EXPONENTS: std::ops::RangeInclusive = -5..=15; +pub(crate) const PYTHON_EXPONENT_DIGITS: usize = 2; +pub(crate) const ORJSON_EXPONENT_DIGITS: usize = 1; +/// CPython's default `sys.get_int_max_str_digits()`, beyond which `int.__repr__` raises. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; +/// Containers nested deeper than this are refused; orjson's own limit is 255 and json's is higher. +pub(crate) const MAX_NESTING_DEPTH: usize = 128; diff --git a/crates/sqlbuild-core/src/json/errors.rs b/crates/sqlbuild-core/src/json/errors.rs new file mode 100644 index 0000000000..0c731dec9a --- /dev/null +++ b/crates/sqlbuild-core/src/json/errors.rs @@ -0,0 +1,14 @@ +//! Values a Python serializer refuses. + +/// A value the chosen serializer refuses, named after the Python failure it reproduces. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JsonEmitError { + /// `json.dumps(..., allow_nan=False)` met `NaN` or an infinity and raised `ValueError`. + NonFiniteFloat, + /// orjson met an integer outside `[-2**63, 2**64 - 1]` and raised `JSONEncodeError`. + IntegerOutOfRange, + /// `json.dumps` met an integer with more digits than Python's string conversion limit. + IntegerTooLong, + /// The value nests deeper than the native emitter writes; callers fall back to Python. + NestingTooDeep, +} diff --git a/crates/sqlbuild-core/src/json/main/dumps.rs b/crates/sqlbuild-core/src/json/main/dumps.rs new file mode 100644 index 0000000000..ccf47da133 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/dumps.rs @@ -0,0 +1,9 @@ +//! Serialize JSON exactly as SQLBuild's Python serializers do. + +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonValue}; + +/// The text `json.dumps(value, ...)` or `orjson.dumps(value, ...).decode()` returns. +pub fn dumps(value: &JsonValue, dialect: &JsonDialect) -> Result { + crate::json::_helpers::emitter::emit(value, dialect) +} diff --git a/crates/sqlbuild-core/src/json/main/mod.rs b/crates/sqlbuild-core/src/json/main/mod.rs new file mode 100644 index 0000000000..ca5e541504 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/mod.rs @@ -0,0 +1 @@ +pub mod dumps; diff --git a/crates/sqlbuild-core/src/json/mod.rs b/crates/sqlbuild-core/src/json/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-core/src/json/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/json/models.rs b/crates/sqlbuild-core/src/json/models.rs new file mode 100644 index 0000000000..351ec48f0a --- /dev/null +++ b/crates/sqlbuild-core/src/json/models.rs @@ -0,0 +1,138 @@ +//! JSON values and the Python serializer configurations the emitter reproduces. + +/// A JSON value with Python's value domain; objects keep insertion order and unique keys. +#[derive(Clone, Debug, PartialEq)] +pub enum JsonValue { + Null, + Bool(bool), + Integer(JsonInteger), + Float(f64), + String(String), + Array(Vec), + Object(Vec<(String, JsonValue)>), +} + +/// An arbitrary-precision integer as canonical decimal text without leading zeros or `-0`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JsonInteger { + decimal: String, +} + +/// Which Python serializer, with which options, to reproduce. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JsonDialect { + /// The standard library's `json.dumps`. + Stdlib(StdlibJsonOptions), + /// `orjson.dumps`. + Orjson(OrjsonOptions), +} + +/// Options of `json.dumps`, with the same meaning and defaults. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StdlibJsonOptions { + /// `indent`, as the string repeated once per nesting level (`indent=2` is two spaces). + pub indent: Option, + /// First element of `separators`. + pub item_separator: String, + /// Second element of `separators`. + pub key_separator: String, + pub ensure_ascii: bool, + pub sort_keys: bool, + pub allow_nan: bool, +} + +/// The `orjson` options SQLBuild uses; every other option is off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct OrjsonOptions { + /// `orjson.OPT_INDENT_2`. + pub indent_2: bool, + /// `orjson.OPT_SORT_KEYS`. + pub sort_keys: bool, +} + +impl StdlibJsonOptions { + /// The options of `json.dumps(value, indent=indent)`, with Python's default separators. + pub fn new(indent: Option) -> Self { + let item_separator = if indent.is_some() { "," } else { ", " }; + Self { + indent: indent.map(|width| " ".repeat(width)), + item_separator: item_separator.to_owned(), + key_separator: ": ".to_owned(), + ensure_ascii: true, + sort_keys: false, + allow_nan: true, + } + } + + /// The same options with `separators=(item_separator, key_separator)`. + pub fn with_separators(self, item_separator: &str, key_separator: &str) -> Self { + Self { + item_separator: item_separator.to_owned(), + key_separator: key_separator.to_owned(), + ..self + } + } +} + +impl JsonInteger { + /// Parse decimal text with an optional sign and leading zeros, as Python's `int()` does. + pub fn parse(text: &str) -> Option { + let (negative, digits) = match text.as_bytes().first()? { + b'-' => (true, &text[1..]), + b'+' => (false, &text[1..]), + _ => (false, text), + }; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let significant = digits.trim_start_matches('0'); + let decimal = match (significant.is_empty(), negative) { + (true, _) => "0".to_owned(), + (false, true) => format!("-{significant}"), + (false, false) => significant.to_owned(), + }; + Some(Self { decimal }) + } + + /// The canonical decimal text. + pub fn as_str(&self) -> &str { + &self.decimal + } + + /// The value as an `i128`, or `None` when it needs more than 128 bits. + pub fn to_i128(&self) -> Option { + let (sign, digits) = match self.decimal.strip_prefix('-') { + Some(digits) => (-1, digits), + None => (1, self.decimal.as_str()), + }; + digits.bytes().try_fold(0_i128, |total, digit| { + total + .checked_mul(10)? + .checked_add(sign * i128::from(digit - b'0')) + }) + } +} + +impl From for JsonInteger { + fn from(value: i64) -> Self { + Self { + decimal: value.to_string(), + } + } +} + +impl From for JsonInteger { + fn from(value: u64) -> Self { + Self { + decimal: value.to_string(), + } + } +} + +impl From for JsonInteger { + fn from(value: i128) -> Self { + Self { + decimal: value.to_string(), + } + } +} diff --git a/crates/sqlbuild-core/src/json/tests.rs b/crates/sqlbuild-core/src/json/tests.rs new file mode 100644 index 0000000000..49c3d52143 --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dumps.rs"] +mod dumps; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/json/tests/helpers.rs b/crates/sqlbuild-core/src/json/tests/helpers.rs new file mode 100644 index 0000000000..25229c37ff --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/helpers.rs @@ -0,0 +1,57 @@ +use crate::json::models::{JsonInteger, JsonValue}; + +fn string(text: &str) -> JsonValue { + JsonValue::String(text.to_owned()) +} + +fn integer(text: &str) -> JsonValue { + JsonValue::Integer(JsonInteger::parse(text).expect("test integers are decimal")) +} + +/// A value exercising escapes, big integers, float layouts, empty containers and key order. +pub(super) fn sample_document(largest_integer: &str) -> JsonValue { + let floats = [0.1, -0.0, 1e16, 1e-5, 2.5e-7, 1e15, 100.0]; + JsonValue::Object(vec![ + ("name".to_owned(), string("café ☕ 😀")), + ("control".to_owned(), string("\u{0}\u{1f}\u{7f}\t\n\"\\/")), + ( + "ints".to_owned(), + JsonValue::Array(vec![ + integer("0"), + integer("-1"), + integer("9223372036854775808"), + integer(largest_integer), + ]), + ), + ( + "floats".to_owned(), + JsonValue::Array(floats.into_iter().map(JsonValue::Float).collect()), + ), + ( + "empty".to_owned(), + JsonValue::Array(vec![JsonValue::Object(vec![]), JsonValue::Array(vec![])]), + ), + ("z".to_owned(), JsonValue::Null), + ("a".to_owned(), JsonValue::Bool(true)), + ]) +} + +pub(super) fn non_finite_floats() -> JsonValue { + JsonValue::Array(vec![ + JsonValue::Float(f64::NAN), + JsonValue::Float(f64::INFINITY), + JsonValue::Float(f64::NEG_INFINITY), + ]) +} + +/// An integer written with `count` decimal digits. +pub(super) fn digits(count: usize) -> JsonValue { + integer(&"9".repeat(count)) +} + +/// `depth` arrays nested inside one another. +pub(super) fn nested_arrays(depth: usize) -> JsonValue { + (1..depth).fold(JsonValue::Array(Vec::new()), |inner, _| { + JsonValue::Array(vec![inner]) + }) +} diff --git a/crates/sqlbuild-core/src/json/tests/test_dumps.rs b/crates/sqlbuild-core/src/json/tests/test_dumps.rs new file mode 100644 index 0000000000..2d1584eb5d --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/test_dumps.rs @@ -0,0 +1,267 @@ +use crate::json::_helpers::floats::python_repr; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, OrjsonOptions, StdlibJsonOptions}; +use crate::json::tests::helpers::{digits, nested_arrays, non_finite_floats, sample_document}; +use crate::json::tests::test_types::{DumpsTestCase, FloatReprTestCase, IntegerParseTestCase}; + +const BIG: &str = "1180591620717411303424"; +const U64_MAX: &str = "18446744073709551615"; + +#[test] +fn given_python_serializer_options_when_dumping_then_output_matches_python_bytes() { + let stdlib_compact_sorted = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let stdlib_unicode_indented = StdlibJsonOptions { + ensure_ascii: false, + sort_keys: true, + ..StdlibJsonOptions::new(Some(2)) + }; + let test_cases = [ + DumpsTestCase { + description: "json.dumps defaults", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Ok( + r#"{"name": "caf\u00e9 \u2615 \ud83d\ude00", "control": "\u0000\u001f\u007f\t\n\"\\/", "ints": [0, -1, 9223372036854775808, 1180591620717411303424], "floats": [0.1, -0.0, 1e+16, 1e-05, 2.5e-07, 1000000000000000.0, 100.0], "empty": [{}, []], "z": null, "a": true}"#, + ), + }, + DumpsTestCase { + description: "json.dumps indent=2", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + expected_text: Ok( + "{\n \"name\": \"caf\\u00e9 \\u2615 \\ud83d\\ude00\",\n \"control\": \"\\u0000\\u001f\\u007f\\t\\n\\\"\\\\/\",\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 1180591620717411303424\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 1e-05,\n 2.5e-07,\n 1000000000000000.0,\n 100.0\n ],\n \"empty\": [\n {},\n []\n ],\n \"z\": null,\n \"a\": true\n}", + ), + }, + DumpsTestCase { + description: "json.dumps compact sorted ASCII", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(stdlib_compact_sorted), + expected_text: Ok( + r#"{"a":true,"control":"\u0000\u001f\u007f\t\n\"\\/","empty":[{},[]],"floats":[0.1,-0.0,1e+16,1e-05,2.5e-07,1000000000000000.0,100.0],"ints":[0,-1,9223372036854775808,1180591620717411303424],"name":"caf\u00e9 \u2615 \ud83d\ude00","z":null}"#, + ), + }, + DumpsTestCase { + description: "json.dumps ensure_ascii=False indent=2 sorted", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(stdlib_unicode_indented), + expected_text: Ok( + "{\n \"a\": true,\n \"control\": \"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\n \"empty\": [\n {},\n []\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 1e-05,\n 2.5e-07,\n 1000000000000000.0,\n 100.0\n ],\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 1180591620717411303424\n ],\n \"name\": \"café ☕ 😀\",\n \"z\": null\n}", + ), + }, + DumpsTestCase { + description: "orjson OPT_INDENT_2", + value: sample_document(U64_MAX), + dialect: JsonDialect::Orjson(OrjsonOptions { + indent_2: true, + sort_keys: false, + }), + expected_text: Ok( + "{\n \"name\": \"café ☕ 😀\",\n \"control\": \"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 18446744073709551615\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 0.00001,\n 2.5e-7,\n 1000000000000000.0,\n 100.0\n ],\n \"empty\": [\n {},\n []\n ],\n \"z\": null,\n \"a\": true\n}", + ), + }, + DumpsTestCase { + description: "orjson OPT_SORT_KEYS", + value: sample_document(U64_MAX), + dialect: JsonDialect::Orjson(OrjsonOptions { + indent_2: false, + sort_keys: true, + }), + expected_text: Ok( + "{\"a\":true,\"control\":\"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\"empty\":[{},[]],\"floats\":[0.1,-0.0,1e+16,0.00001,2.5e-7,1000000000000000.0,100.0],\"ints\":[0,-1,9223372036854775808,18446744073709551615],\"name\":\"café ☕ 😀\",\"z\":null}", + ), + }, + DumpsTestCase { + description: "orjson rejects integers beyond 64 bits", + value: sample_document(BIG), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Err(JsonEmitError::IntegerOutOfRange), + }, + DumpsTestCase { + description: "json.dumps writes NaN and infinities", + value: non_finite_floats(), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Ok("[NaN, Infinity, -Infinity]"), + }, + DumpsTestCase { + description: "json.dumps allow_nan=False rejects NaN", + value: non_finite_floats(), + dialect: JsonDialect::Stdlib(StdlibJsonOptions { + allow_nan: false, + ..StdlibJsonOptions::new(None) + }), + expected_text: Err(JsonEmitError::NonFiniteFloat), + }, + DumpsTestCase { + description: "orjson writes non-finite floats as null", + value: non_finite_floats(), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Ok("[null,null,null]"), + }, + DumpsTestCase { + description: "json.dumps refuses integers beyond Python's string conversion limit", + value: digits(4_301), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Err(JsonEmitError::IntegerTooLong), + }, + DumpsTestCase { + description: "deeply nested values are refused instead of exhausting the stack", + value: nested_arrays(1_000), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + expected_text: Err(JsonEmitError::NestingTooDeep), + }, + DumpsTestCase { + description: "orjson refuses deeply nested values too", + value: nested_arrays(130), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Err(JsonEmitError::NestingTooDeep), + }, + DumpsTestCase { + description: "nesting within the limit is written", + value: nested_arrays(3), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Ok("[[[]]]"), + }, + ]; + for test_case in test_cases { + let actual = dumps(&test_case.value, &test_case.dialect); + assert_eq!( + actual.as_deref().map_err(|error| *error), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_floats_when_formatting_python_repr_then_text_matches_cpython() { + let test_cases = [ + FloatReprTestCase { + description: "0.0", + value: 0.0, + expected_repr: "0.0", + }, + FloatReprTestCase { + description: "-0.0", + value: -0.0, + expected_repr: "-0.0", + }, + FloatReprTestCase { + description: "0.0001", + value: 1e-4, + expected_repr: "0.0001", + }, + FloatReprTestCase { + description: "1e-05", + value: 1e-5, + expected_repr: "1e-05", + }, + FloatReprTestCase { + description: "1.5e-300", + value: 1.5e-300, + expected_repr: "1.5e-300", + }, + FloatReprTestCase { + description: "5e-324", + value: 5e-324, + expected_repr: "5e-324", + }, + FloatReprTestCase { + description: "1.2345678901234568e+17", + value: 123_456_789_012_345_680.0, + expected_repr: "1.2345678901234568e+17", + }, + FloatReprTestCase { + description: "9007199254740992.0", + value: 9_007_199_254_740_993.0, + expected_repr: "9007199254740992.0", + }, + FloatReprTestCase { + description: "1.7976931348623157e+308", + value: f64::MAX, + expected_repr: "1.7976931348623157e+308", + }, + FloatReprTestCase { + description: "0.30000000000000004", + value: 0.30000000000000004, + expected_repr: "0.30000000000000004", + }, + FloatReprTestCase { + description: "nan", + value: f64::NAN, + expected_repr: "nan", + }, + FloatReprTestCase { + description: "exact halfway digit rounds half to even", + value: 73_720_622_874_222.62, + expected_repr: "73720622874222.62", + }, + FloatReprTestCase { + description: "-inf", + value: f64::NEG_INFINITY, + expected_repr: "-inf", + }, + ]; + for test_case in test_cases { + assert_eq!( + python_repr(test_case.value), + test_case.expected_repr, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_python_integer_text_when_parsing_then_text_is_canonical() { + let test_cases = [ + IntegerParseTestCase { + description: "leading zeros are dropped", + text: "007", + expected_decimal: Some("7"), + }, + IntegerParseTestCase { + description: "negative zero is zero", + text: "-0", + expected_decimal: Some("0"), + }, + IntegerParseTestCase { + description: "plus sign is dropped", + text: "+42", + expected_decimal: Some("42"), + }, + IntegerParseTestCase { + description: "negative keeps its sign", + text: "-000123", + expected_decimal: Some("-123"), + }, + IntegerParseTestCase { + description: "letters are rejected", + text: "x", + expected_decimal: None, + }, + IntegerParseTestCase { + description: "empty text is rejected", + text: "", + expected_decimal: None, + }, + IntegerParseTestCase { + description: "bare sign is rejected", + text: "-", + expected_decimal: None, + }, + ]; + for test_case in test_cases { + let parsed = JsonInteger::parse(test_case.text); + assert_eq!( + parsed.as_ref().map(JsonInteger::as_str), + test_case.expected_decimal, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/json/tests/test_types.rs b/crates/sqlbuild-core/src/json/tests/test_types.rs new file mode 100644 index 0000000000..2ce319a3b0 --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/test_types.rs @@ -0,0 +1,21 @@ +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonValue}; + +pub(super) struct DumpsTestCase { + pub(super) description: &'static str, + pub(super) value: JsonValue, + pub(super) dialect: JsonDialect, + pub(super) expected_text: Result<&'static str, JsonEmitError>, +} + +pub(super) struct FloatReprTestCase { + pub(super) description: &'static str, + pub(super) value: f64, + pub(super) expected_repr: &'static str, +} + +pub(super) struct IntegerParseTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_decimal: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index dc11a3fe2a..d94e5102aa 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -1,8 +1,9 @@ -//! Shared foundations of the SQLBuild native workspace: identities, diagnostics, text and typed values. +//! Shared foundations of the SQLBuild native workspace: identities, diagnostics, text and JSON. #![forbid(unsafe_code)] pub mod constants; +pub mod json; pub mod models; pub mod panics; pub mod text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs index 8bd0052338..95f89aec8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs @@ -593,6 +593,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; + crate::bindings::_helpers::oracles::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 83143c2dbf..d1e72c2cfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,5 +3,6 @@ mod catalog; pub(crate) mod conversions; pub(crate) mod functions; mod normalization_results; +mod oracles; pub(crate) mod panics; mod positions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs new file mode 100644 index 0000000000..8b1ad68dbc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -0,0 +1,137 @@ +//! Internal hooks that let Python tests compare native foundations with Python byte for byte. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{ + JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, +}; +use sqlbuild_core::text::main::decode_python_text::decode_python_text; +use sqlbuild_core::text::models::LineIndex; + +fn oracle_error(message: impl std::fmt::Display) -> PyErr { + PyValueError::new_err(message.to_string()) +} + +fn text_field<'value>(value: &'value Value, key: &str) -> PyResult<&'value str> { + value[key] + .as_str() + .ok_or_else(|| oracle_error(format!("expected string field {key}"))) +} + +fn flag(value: &Value, key: &str) -> PyResult { + value[key] + .as_bool() + .ok_or_else(|| oracle_error(format!("expected boolean field {key}"))) +} + +fn dialect(spec: &Value) -> PyResult { + if let Some(options) = spec.get("orjson") { + return Ok(JsonDialect::Orjson(OrjsonOptions { + indent_2: flag(options, "indent_2")?, + sort_keys: flag(options, "sort_keys")?, + })); + } + Ok(JsonDialect::Stdlib(StdlibJsonOptions { + indent: spec["indent"].as_str().map(str::to_owned), + item_separator: text_field(&spec["separators"], "item")?.to_owned(), + key_separator: text_field(&spec["separators"], "key")?.to_owned(), + ensure_ascii: flag(spec, "ensure_ascii")?, + sort_keys: flag(spec, "sort_keys")?, + allow_nan: flag(spec, "allow_nan")?, + })) +} + +fn float_from_bits(bits: &Value) -> PyResult { + let text = bits + .as_str() + .ok_or_else(|| oracle_error("expected float bits"))?; + text.parse::() + .map(f64::from_bits) + .map_err(oracle_error) +} + +/// Decode `[tag, payload]`, the tagged form the Python oracle tests write. +fn tagged_value(tagged: &Value) -> PyResult { + let payload = &tagged[1]; + match tagged[0].as_str() { + Some("null") => Ok(JsonValue::Null), + Some("bool") => payload + .as_bool() + .map(JsonValue::Bool) + .ok_or_else(|| oracle_error("expected boolean")), + Some("int") => payload + .as_str() + .and_then(JsonInteger::parse) + .map(JsonValue::Integer) + .ok_or_else(|| oracle_error("expected decimal integer")), + Some("float") => float_from_bits(payload).map(JsonValue::Float), + Some("str") => payload + .as_str() + .map(|text| JsonValue::String(text.to_owned())) + .ok_or_else(|| oracle_error("expected string")), + Some("list") => tagged_items(payload)? + .iter() + .map(tagged_value) + .collect::>>() + .map(JsonValue::Array), + Some("dict") => tagged_items(payload)? + .iter() + .map(|entry| { + let key = entry[0] + .as_str() + .ok_or_else(|| oracle_error("expected string key"))?; + Ok((key.to_owned(), tagged_value(&entry[1])?)) + }) + .collect::>>() + .map(JsonValue::Object), + _ => Err(oracle_error("unknown tagged value")), + } +} + +fn tagged_items(payload: &Value) -> PyResult<&Vec> { + payload + .as_array() + .ok_or_else(|| oracle_error("expected tagged items")) +} + +/// Serialize a tagged value natively, or return `error:` for a refused value. +#[pyfunction] +fn _oracle_json_dumps(dialect_json: &str, value_json: &str) -> PyResult { + let spec: Value = serde_json::from_str(dialect_json).map_err(oracle_error)?; + let tagged: Value = serde_json::from_str(value_json).map_err(oracle_error)?; + Ok(dumps(&tagged_value(&tagged)?, &dialect(&spec)?) + .unwrap_or_else(|error| format!("error:{error:?}"))) +} + +type OraclePosition = Option<(usize, usize, usize)>; + +fn oracle_position(index: &LineIndex<'_>, byte_offset: usize) -> OraclePosition { + let position = index.position_at_byte(byte_offset)?; + Some((position.char_offset, position.line, position.column)) +} + +/// Decode authored bytes and report `(char_offset, line, column)` for each byte offset. +#[pyfunction] +fn _oracle_text_positions( + data: &[u8], + byte_offsets: Vec, +) -> PyResult<(String, Vec)> { + let text = decode_python_text(data).map_err(|error| oracle_error(error.valid_up_to))?; + let positions = { + let index = LineIndex::new(&text); + byte_offsets + .into_iter() + .map(|offset| oracle_position(&index, offset)) + .collect() + }; + Ok((text, positions)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; + Ok(()) +} diff --git a/pyproject.toml b/pyproject.toml index 1b03327fb7..2cee655b4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.12" dependencies = [ "duckdb>=1.3.2", "filelock>=3.20.3", - "orjson>=3.10.0", + "orjson>=3.11.7", "polyglot-sql-chio==0.13.8", "pydantic-settings>=2,<3", "PyYAML>=6.0.2", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3eb370a343..c98c22bfaf 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -49,3 +49,9 @@ def substitute_static_project_vars( def extract_static_sql_references( sql: str, ) -> list[tuple[str, str, str | None, int | None]] | None: ... + +# Internal oracle hooks for tests that compare native foundations with Python; not an API. +def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... +def _oracle_text_positions( + data: bytes, byte_offsets: list[int] +) -> tuple[str, list[tuple[int, int, int] | None]]: ... diff --git a/tests/integration/src/sqlbuild/compiler/_test_types.py b/tests/integration/src/sqlbuild/compiler/_test_types.py new file mode 100644 index 0000000000..f17956cecf --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/_test_types.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JsonOracleTestCase: + """One Python serializer configuration compared with the native emitter on seeded values.""" + + description: str + serializer: str + seed: int + case_count: int + indent: int | str | None = None + separators: tuple[str, str] | None = None + ensure_ascii: bool = True + sort_keys: bool = False + allow_nan: bool = True + orjson_option: int = 0 + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class TextOracleTestCase: + """Seeded authored files whose native decoding and positions are compared with Python.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class OrjsonFloatLayoutTestCase: + """A float whose orjson text depends on the orjson version the native emitter reproduces.""" + + description: str + value: float + expected_text: str diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py new file mode 100644 index 0000000000..67f935b588 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -0,0 +1,301 @@ +"""Seeded generators and Python-side results shared by the native oracle tests.""" + +from __future__ import annotations + +import json +import math +import random +import struct +from collections.abc import Callable +from contextlib import suppress +from itertools import compress +from operator import ne +from pathlib import Path + +import orjson + +from sqlbuild import _native +from tests.integration.src.sqlbuild.compiler._test_types import JsonOracleTestCase + +NASTY_FLOATS: tuple[float, ...] = ( + 0.0, + -0.0, + math.nan, + math.inf, + -math.inf, + 5e-324, + -5e-324, + 2.2250738585072014e-308, + 1.7976931348623157e308, + 1e16, + 9999999999999998.0, + 1e15, + 1e-4, + 1e-5, + 9.999999999999999e-5, + 0.1, + 0.30000000000000004, + 1 / 3, + 2.0**53, + 2.0**53 + 2, + 1e22, + 1e-7, + 1.5e-6, + 100.0, + -123456.789, +) +NASTY_INTEGERS: tuple[int, ...] = ( + 0, + -1, + 2**31, + 2**53 + 1, + 2**63 - 1, + 2**63, + -(2**63), + -(2**63) - 1, + 2**64 - 1, + 2**64, + 10**40, + -(10**40), +) +CHARACTER_RANGES: tuple[tuple[int, int], ...] = ( + (0x20, 0x7E), + (0x20, 0x7E), + (0x20, 0x7E), + (0x00, 0x1F), + (0x7F, 0xA0), + (0xA1, 0x24F), + (0x2000, 0x206F), + (0x4E00, 0x4E80), + (0xE000, 0xE010), + (0xFFF0, 0xFFFF), + (0x1F600, 0x1F64F), + (0x10FFF0, 0x10FFFF), + (0x22, 0x22), + (0x5C, 0x5C), + (0x2028, 0x2029), + (0xFEFF, 0xFEFF), +) +LINE_BREAKS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", "\r\r\n") +INVALID_UTF8: tuple[bytes, ...] = (b"\xff", b"\xc3", b"\xed\xa0\x80", b"\xf4\x90\x80\x80") +MAX_JSON_DEPTH: int = 4 + + +def random_character(*, rng: random.Random) -> str: + """Return one code point from a range chosen to stress escaping and offsets.""" + + low, high = rng.choice(CHARACTER_RANGES) + return chr(rng.randint(low, high)) + + +def random_text(*, rng: random.Random, max_length: int) -> str: + """Return a string mixing ASCII, controls, non-BMP and other awkward code points.""" + + return "".join(random_character(rng=rng) for _ in range(rng.randint(0, max_length))) + + +def _float_from_bits(rng: random.Random) -> float: + return struct.unpack(" float: + mantissa: int = rng.randint(1, 10 ** rng.randint(1, 17)) + return float(f"{rng.choice('-+')}{mantissa}e{rng.randint(-340, 300)}") + + +FLOAT_GENERATORS: tuple[Callable[[random.Random], float], ...] = ( + lambda rng: rng.choice(NASTY_FLOATS), + _float_from_bits, + _float_from_decimal, +) +INTEGER_GENERATORS: tuple[Callable[[random.Random], int], ...] = ( + lambda rng: rng.choice(NASTY_INTEGERS) + rng.randint(-2, 2), + lambda rng: rng.randint(-(2 ** rng.randint(1, 140)), 2 ** rng.randint(1, 140)), +) + + +def random_float(*, rng: random.Random) -> float: + """Return a float from the nasty pool, random bits, or a random decimal.""" + + return rng.choice(FLOAT_GENERATORS)(rng) + + +def random_integer(*, rng: random.Random) -> int: + """Return an integer near the 32- and 64-bit boundaries or of arbitrary size.""" + + return rng.choice(INTEGER_GENERATORS)(rng) + + +def _json_list(rng: random.Random, depth: int) -> object: + return [random_json_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 4))] + + +def _json_dict(rng: random.Random, depth: int) -> object: + return { + random_text(rng=rng, max_length=6): random_json_value(rng=rng, depth=depth - 1) + for _ in range(rng.randint(0, 4)) + } + + +JSON_SCALAR_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + lambda rng, depth: None, + lambda rng, depth: rng.random() < 0.5, + lambda rng, depth: random_integer(rng=rng), + lambda rng, depth: random_float(rng=rng), + lambda rng, depth: random_text(rng=rng, max_length=12), +) +JSON_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + *JSON_SCALAR_GENERATORS, + _json_list, + _json_dict, + _json_list, + _json_dict, +) + + +def random_json_value(*, rng: random.Random, depth: int) -> object: + """Return a random JSON-compatible Python value, preferring containers above the leaves.""" + + available: int = len(JSON_SCALAR_GENERATORS) + 4 * min(depth, 1) + return rng.choice(JSON_GENERATORS[:available])(rng, depth) + + +def _float_bits(value: float) -> str: + return str(struct.unpack(" list[object]: + """Return the tagged form the native JSON oracle hook decodes.""" + + return TAGGERS[type(value)](value) + + +def python_json_text(*, value: object, test_case: JsonOracleTestCase) -> str: + """Return Python's serializer output, or the name of the failure it raises.""" + + outcome: list[str] = [f"error:{SERIALIZER_FAILURES[test_case.serializer]}"] + with suppress(ValueError, TypeError): + outcome[0] = PYTHON_SERIALIZERS[test_case.serializer](value, test_case) + return outcome[0] + + +def _stdlib_dumps(value: object, test_case: JsonOracleTestCase) -> str: + return json.dumps( + value, + indent=test_case.indent, + separators=test_case.separators, + ensure_ascii=test_case.ensure_ascii, + sort_keys=test_case.sort_keys, + allow_nan=test_case.allow_nan, + ) + + +PYTHON_SERIALIZERS: dict[str, Callable[[object, JsonOracleTestCase], str]] = { + "stdlib": _stdlib_dumps, + "orjson": lambda value, test_case: orjson.dumps(value, option=test_case.orjson_option).decode(), +} +SERIALIZER_FAILURES: dict[str, str] = {"stdlib": "NonFiniteFloat", "orjson": "IntegerOutOfRange"} + + +INDENT_TEXT: dict[type, Callable[[object], object]] = { + int: lambda width: " " * width, + str: lambda text: text, + type(None): lambda nothing: nothing, +} + + +def native_json_text(*, value: object, test_case: JsonOracleTestCase) -> str: + """Return the native emitter's output for the same value and configuration.""" + + default_separators: tuple[str, str] = ((", ", ": "), (",", ": "))[test_case.indent is not None] + item_separator, key_separator = test_case.separators or default_separators + indent: object = INDENT_TEXT[type(test_case.indent)](test_case.indent) + specs: dict[str, dict[str, object]] = { + "stdlib": { + "indent": indent, + "separators": {"item": item_separator, "key": key_separator}, + "ensure_ascii": test_case.ensure_ascii, + "sort_keys": test_case.sort_keys, + "allow_nan": test_case.allow_nan, + }, + "orjson": { + "orjson": { + "indent_2": bool(test_case.orjson_option & orjson.OPT_INDENT_2), + "sort_keys": bool(test_case.orjson_option & orjson.OPT_SORT_KEYS), + } + }, + } + spec: dict[str, object] = specs[test_case.serializer] + return _native._oracle_json_dumps(json.dumps(spec), json.dumps(tagged(value))) + + +def mismatches( + *, inputs: list[object], expected: list[object], actual: list[object] +) -> list[tuple[object, object, object]]: + """Return the first few `(input, expected, actual)` triples that differ.""" + + return list(compress(zip(inputs, expected, actual, strict=True), map(ne, expected, actual)))[:3] + + +def authored_bytes(*, rng: random.Random) -> bytes: + """Return valid UTF-8 text with every newline convention, tabs and non-ASCII.""" + + parts: list[str] = [ + random_text(rng=rng, max_length=8) + rng.choice(LINE_BREAKS) + for _ in range(rng.randint(0, 12)) + ] + return "".join(parts).encode("utf-8") + + +def invalid_authored_bytes(*, rng: random.Random) -> bytes: + """Return authored bytes with one invalid UTF-8 sequence inserted.""" + + data: bytes = authored_bytes(rng=rng) + cut: int = rng.randint(0, len(data)) + return data[:cut] + rng.choice(INVALID_UTF8) + data[cut:] + + +def python_positions(*, text: str, char_offsets: list[int]) -> list[tuple[int, int, int]]: + """Return `(char_offset, line, column)` with SQLBuild's Python line and column arithmetic.""" + + return [ + ( + offset, + text.count("\n", 0, offset) + 1, + offset - (text.rfind("\n", 0, offset) + 1) + 1, + ) + for offset in char_offsets + ] + + +def random_char_offsets(*, rng: random.Random, text: str) -> list[int]: + """Return sorted random code-point offsets into `text`, including its end.""" + + return sorted(rng.randint(0, len(text)) for _ in range(8)) + + +def utf8_offsets(*, text: str, char_offsets: list[int]) -> list[int]: + """Return the UTF-8 byte offset of each code-point offset into `text`.""" + + return [len(text[:offset].encode("utf-8")) for offset in char_offsets] + + +def python_read_text(*, path: Path, data: bytes) -> str: + """Write `data` and read it back the way SQLBuild reads authored files.""" + + path.write_bytes(data) + return path.read_text(encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py new file mode 100644 index 0000000000..749b212c47 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py @@ -0,0 +1,149 @@ +"""Compare the native JSON emitter with Python's serializers on seeded generated values.""" + +from __future__ import annotations + +import random + +import orjson +import pytest + +from tests.integration.src.sqlbuild.compiler._test_types import ( + JsonOracleTestCase, + OrjsonFloatLayoutTestCase, +) +from tests.integration.src.sqlbuild.compiler.helpers import ( + MAX_JSON_DEPTH, + mismatches, + native_json_text, + python_json_text, + random_json_value, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JsonOracleTestCase( + description="json.dumps defaults", serializer="stdlib", seed=11, case_count=1500 + ), + JsonOracleTestCase( + description="json.dumps indent=2 for manifests and DAG output", + serializer="stdlib", + seed=12, + case_count=1500, + indent=2, + ), + JsonOracleTestCase( + description="json.dumps compact sorted ASCII for identities and caches", + serializer="stdlib", + seed=13, + case_count=1500, + separators=(",", ":"), + sort_keys=True, + ), + JsonOracleTestCase( + description="json.dumps Unicode sorted with indent", + serializer="stdlib", + seed=14, + case_count=1500, + indent=2, + ensure_ascii=False, + sort_keys=True, + ), + JsonOracleTestCase( + description="json.dumps tab indent with custom separators", + serializer="stdlib", + seed=15, + case_count=1500, + indent="\t", + separators=(", ", " = "), + ), + JsonOracleTestCase( + description="json.dumps allow_nan=False rejects non-finite floats", + serializer="stdlib", + seed=16, + case_count=1500, + separators=(",", ":"), + allow_nan=False, + ), + JsonOracleTestCase( + description="orjson defaults", serializer="orjson", seed=21, case_count=1500 + ), + JsonOracleTestCase( + description="orjson OPT_INDENT_2 for the compile payload", + serializer="orjson", + seed=22, + case_count=1500, + orjson_option=orjson.OPT_INDENT_2, + ), + JsonOracleTestCase( + description="orjson OPT_SORT_KEYS", + serializer="orjson", + seed=23, + case_count=1500, + orjson_option=orjson.OPT_SORT_KEYS, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_values_when_dumping_natively_then_bytes_match_python_serializer( + test_case: JsonOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + values: list[object] = [ + random_json_value(rng=rng, depth=MAX_JSON_DEPTH) for _ in range(test_case.case_count) + ] + + expected: list[object] = [ + python_json_text(value=value, test_case=test_case) for value in values + ] + actual: list[object] = [native_json_text(value=value, test_case=test_case) for value in values] + + assert mismatches(inputs=values, expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + OrjsonFloatLayoutTestCase( + description="positive exponents carry a plus sign from orjson 3.11.7", + value=1e16, + expected_text="1e+16", + ), + OrjsonFloatLayoutTestCase( + description="large positive exponents carry a plus sign", + value=1.5e300, + expected_text="1.5e+300", + ), + OrjsonFloatLayoutTestCase( + description="negative exponents keep their sign without padding", + value=1e-6, + expected_text="1e-6", + ), + OrjsonFloatLayoutTestCase( + description="small floats stay fixed down to five decimal places", + value=1e-5, + expected_text="0.00001", + ), + ], + ids=lambda case: case.description, +) +def test_given_installed_orjson_when_dumping_exponent_floats_then_text_matches_native_layout( + test_case: OrjsonFloatLayoutTestCase, +) -> None: + orjson_case: JsonOracleTestCase = JsonOracleTestCase( + description=test_case.description, serializer="orjson", seed=0, case_count=1 + ) + + installed: str = orjson.dumps(test_case.value).decode() + native: str = native_json_text(value=test_case.value, test_case=orjson_case) + + assert (installed, native) == (test_case.expected_text, test_case.expected_text), ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py new file mode 100644 index 0000000000..dcde8880bb --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py @@ -0,0 +1,86 @@ +"""Compare native text decoding and positions with Python's read_text and column arithmetic.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from sqlbuild import _native +from tests.integration.src.sqlbuild.compiler._test_types import TextOracleTestCase +from tests.integration.src.sqlbuild.compiler.helpers import ( + authored_bytes, + invalid_authored_bytes, + mismatches, + python_positions, + python_read_text, + random_char_offsets, + utf8_offsets, +) + + +@pytest.mark.parametrize( + "test_case", + [ + TextOracleTestCase( + description="CRLF, CR, tabs and non-ASCII before every offset", seed=31, case_count=400 + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_files_when_decoding_natively_then_text_and_positions_match_python( + test_case: TextOracleTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[bytes] = [authored_bytes(rng=rng) for _ in range(test_case.case_count)] + texts: list[str] = [python_read_text(path=tmp_path / "model.sql", data=data) for data in files] + char_offsets: list[list[int]] = [random_char_offsets(rng=rng, text=text) for text in texts] + byte_offsets: list[list[int]] = [ + utf8_offsets(text=text, char_offsets=offsets) + for text, offsets in zip(texts, char_offsets, strict=True) + ] + + expected: list[object] = [ + (text, python_positions(text=text, char_offsets=offsets)) + for text, offsets in zip(texts, char_offsets, strict=True) + ] + actual: list[object] = [ + _native._oracle_text_positions(data, offsets) + for data, offsets in zip(files, byte_offsets, strict=True) + ] + + assert mismatches(inputs=list(files), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [TextOracleTestCase(description="one invalid UTF-8 sequence", seed=32, case_count=100)], + ids=lambda case: case.description, +) +def test_given_invalid_utf8_when_decoding_natively_then_failure_offset_matches_python( + test_case: TextOracleTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + path: Path = tmp_path / "model.sql" + files: list[bytes] = [invalid_authored_bytes(rng=rng) for _ in range(test_case.case_count)] + python_offsets: list[object] = [] + native_offsets: list[object] = [] + + for data in files: + with pytest.raises(UnicodeDecodeError) as python_error: + python_read_text(path=path, data=data) + with pytest.raises(ValueError) as native_error: + _native._oracle_text_positions(data, []) + python_offsets.append(python_error.value.start) + native_offsets.append(int(str(native_error.value))) + + assert mismatches(inputs=list(files), expected=python_offsets, actual=native_offsets) == list( + test_case.expected_mismatches + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/uv.lock b/uv.lock index c66af47428..ba6f77008a 100644 --- a/uv.lock +++ b/uv.lock @@ -4217,7 +4217,7 @@ requires-dist = [ { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.37.0" }, { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.0.0" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, - { name = "orjson", specifier = ">=3.10.0" }, + { name = "orjson", specifier = ">=3.11.7" }, { name = "polyglot-sql-chio", specifier = "==0.13.8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2.0" }, { name = "pydantic-settings", specifier = ">=2,<3" }, From 54111f56213d988c51c66d195c8a80d8e76b23b4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:58:30 +0100 Subject: [PATCH 3/4] feat(native): add a sqlbuild-config crate with tomllib-compatible TOML and project config reading Adds TOML loading that matches tomllib, rejecting TOML 1.1-only syntax, and typed project and local config readers for the fields discovery needs. Any native error defers to the Python loader. Internal oracle hooks compare both against Python on seeded and repository documents. Tables keep tomllib's order of first creation even when an implicit table is declared later, CRLF is normalized and a leading byte order mark is rejected as tomllib does. Container depth and dotted key length are bounded before parsing so deep documents are refused instead of exhausting the stack. tomllib's namespace rules are replayed over the parser events, so dotted keys into arrays of tables or declared tables, headers over dotted or inline tables, and extended inline tables are rejected as tomllib rejects them. --- Cargo.lock | 36 +- Cargo.toml | 5 + crates/README.md | 9 +- crates/sqlbuild-config/Cargo.toml | 18 + crates/sqlbuild-config/src/constants.rs | 11 + crates/sqlbuild-config/src/errors.rs | 37 ++ crates/sqlbuild-config/src/lib.rs | 9 + crates/sqlbuild-config/src/models.rs | 101 +++++ .../src/project/_helpers/fields.rs | 92 +++++ .../src/project/_helpers/files.rs | 35 ++ .../src/project/_helpers/mod.rs | 2 + .../sqlbuild-config/src/project/constants.rs | 9 + .../sqlbuild-config/src/project/main/mod.rs | 2 + .../src/project/main/read_local_config.rs | 40 ++ .../src/project/main/read_project_config.rs | 76 ++++ crates/sqlbuild-config/src/project/mod.rs | 7 + crates/sqlbuild-config/src/project/models.rs | 34 ++ crates/sqlbuild-config/src/project/tests.rs | 6 + .../src/project/tests/helpers.rs | 50 +++ .../src/project/tests/test_read_config.rs | 125 +++++++ .../src/project/tests/test_types.rs | 14 + .../src/toml/_helpers/conversion.rs | 171 +++++++++ .../sqlbuild-config/src/toml/_helpers/mod.rs | 3 + .../src/toml/_helpers/namespaces.rs | 344 +++++++++++++++++ .../src/toml/_helpers/version.rs | 182 +++++++++ .../src/toml/main/load_toml.rs | 19 + crates/sqlbuild-config/src/toml/main/mod.rs | 1 + crates/sqlbuild-config/src/toml/mod.rs | 5 + crates/sqlbuild-config/src/toml/tests.rs | 6 + .../sqlbuild-config/src/toml/tests/helpers.rs | 11 + .../src/toml/tests/test_load_toml.rs | 330 +++++++++++++++++ .../src/toml/tests/test_types.rs | 14 + crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/oracles.rs | 93 ++++- fensu.toml | 1 + src/sqlbuild/_native.pyi | 2 + .../sqlbuild/compiler/discovery/__init__.py | 0 .../compiler/discovery/_test_types.py | 36 ++ .../sqlbuild/compiler/discovery/helpers.py | 346 ++++++++++++++++++ .../test_native_project_config_oracle.py | 48 +++ .../discovery/test_native_toml_oracle.py | 128 +++++++ 41 files changed, 2455 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-config/Cargo.toml create mode 100644 crates/sqlbuild-config/src/constants.rs create mode 100644 crates/sqlbuild-config/src/errors.rs create mode 100644 crates/sqlbuild-config/src/lib.rs create mode 100644 crates/sqlbuild-config/src/models.rs create mode 100644 crates/sqlbuild-config/src/project/_helpers/fields.rs create mode 100644 crates/sqlbuild-config/src/project/_helpers/files.rs create mode 100644 crates/sqlbuild-config/src/project/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/project/constants.rs create mode 100644 crates/sqlbuild-config/src/project/main/mod.rs create mode 100644 crates/sqlbuild-config/src/project/main/read_local_config.rs create mode 100644 crates/sqlbuild-config/src/project/main/read_project_config.rs create mode 100644 crates/sqlbuild-config/src/project/mod.rs create mode 100644 crates/sqlbuild-config/src/project/models.rs create mode 100644 crates/sqlbuild-config/src/project/tests.rs create mode 100644 crates/sqlbuild-config/src/project/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/project/tests/test_read_config.rs create mode 100644 crates/sqlbuild-config/src/project/tests/test_types.rs create mode 100644 crates/sqlbuild-config/src/toml/_helpers/conversion.rs create mode 100644 crates/sqlbuild-config/src/toml/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/toml/_helpers/namespaces.rs create mode 100644 crates/sqlbuild-config/src/toml/_helpers/version.rs create mode 100644 crates/sqlbuild-config/src/toml/main/load_toml.rs create mode 100644 crates/sqlbuild-config/src/toml/main/mod.rs create mode 100644 crates/sqlbuild-config/src/toml/mod.rs create mode 100644 crates/sqlbuild-config/src/toml/tests.rs create mode 100644 crates/sqlbuild-config/src/toml/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/toml/tests/test_load_toml.rs create mode 100644 crates/sqlbuild-config/src/toml/tests/test_types.rs create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py diff --git a/Cargo.lock b/Cargo.lock index df5e1a2b6d..cc5d0beaaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,6 +630,15 @@ dependencies = [ "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-config" +version = "0.138.0" +dependencies = [ + "tempfile", + "toml_edit", + "toml_parser", +] + [[package]] name = "sqlbuild-core" version = "0.138.0" @@ -643,6 +652,7 @@ dependencies = [ "pyo3", "serde_json", "sqlbuild-analysis", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-rules", "sqlbuild-sqltext", @@ -784,7 +794,7 @@ dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow 0.7.15", @@ -799,6 +809,27 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" @@ -892,6 +923,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index b49ec1db4e..2bc33451a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/sqlbuild-core", "crates/sqlbuild-sqltext", + "crates/sqlbuild-config", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -15,6 +16,7 @@ resolver = "2" order = [ "sqlbuild-core", "sqlbuild-sqltext", + "sqlbuild-config", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -59,6 +61,7 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } @@ -67,6 +70,8 @@ sqlparser = { version = "0.58", features = ["visitor"] } syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" toml = "0.9" +toml_edit = { version = "0.25", default-features = false, features = ["parse"] } +toml_parser = "1" walkdir = "2" [workspace.lints.rust] diff --git a/crates/README.md b/crates/README.md index 4bc3c8c67c..37d21326b7 100644 --- a/crates/README.md +++ b/crates/README.md @@ -6,8 +6,9 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| -| `sqlbuild-core` | Constants shared by several crates and the panic boundary that turns an unwinding native failure into a named compiler error. | +| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | +| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -18,9 +19,13 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core ``` +`sqlbuild-config` does not depend on the crates below it today; its place in the order only +fixes which crates may use it. The JSON emitter lives in `sqlbuild-core` so that any layer can +produce text that must equal Python's `json.dumps` output. + Each crate may depend only on crates below it, and may also skip layers. The order is declared in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and `make check-native-layering` (part of `make check-ci`) fails when: diff --git a/crates/sqlbuild-config/Cargo.toml b/crates/sqlbuild-config/Cargo.toml new file mode 100644 index 0000000000..73c43456a5 --- /dev/null +++ b/crates/sqlbuild-config/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqlbuild-config" +version.workspace = true +description = "Project configuration reading for SQLBuild: TOML." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +toml_edit.workspace = true +toml_parser.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-config/src/constants.rs b/crates/sqlbuild-config/src/constants.rs new file mode 100644 index 0000000000..11688decd0 --- /dev/null +++ b/crates/sqlbuild-config/src/constants.rs @@ -0,0 +1,11 @@ +//! Calendar limits of Python's `datetime` types and characters shared by the readers. + +pub(crate) const MIN_YEAR: i64 = 1; +pub(crate) const MAX_YEAR: i64 = 9999; +pub(crate) const MONTHS_PER_YEAR: i64 = 12; +pub(crate) const FEBRUARY: i64 = 2; +pub(crate) const MAX_HOUR: i64 = 23; +pub(crate) const MAX_MINUTE: i64 = 59; +pub(crate) const MAX_SECOND: i64 = 59; +pub(crate) const MAX_MICROSECOND: i64 = 999_999; +pub(crate) const BYTE_ORDER_MARK: char = '\u{feff}'; diff --git a/crates/sqlbuild-config/src/errors.rs b/crates/sqlbuild-config/src/errors.rs new file mode 100644 index 0000000000..972a4c07c9 --- /dev/null +++ b/crates/sqlbuild-config/src/errors.rs @@ -0,0 +1,37 @@ +//! Structured failures of reading configuration files. + +/// Why a configuration document could not be loaded natively. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfigErrorKind { + /// The text is not a valid document; Python's parser rejects it too. + Syntax, + /// The document parses but a value cannot be built, as PyYAML's constructor or `tomllib` reports. + Construct, + /// The document uses a feature the native reader leaves to Python. + Unsupported, + /// A required file does not exist. + Missing, + /// A field SQLBuild reads has the wrong type or an invalid value. + InvalidField, +} + +/// A configuration failure; the Python reader re-parses the file to report its exact message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigError { + pub kind: ConfigErrorKind, + pub message: String, + pub line: Option, + pub column: Option, +} + +impl ConfigError { + /// An error without a position. + pub fn new(kind: ConfigErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + line: None, + column: None, + } + } +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs new file mode 100644 index 0000000000..2ba75e1044 --- /dev/null +++ b/crates/sqlbuild-config/src/lib.rs @@ -0,0 +1,9 @@ +//! Project configuration reading: `tomllib`-compatible TOML. + +#![forbid(unsafe_code)] + +pub(crate) mod constants; +pub mod errors; +pub mod models; +pub mod project; +pub mod toml; diff --git a/crates/sqlbuild-config/src/models.rs b/crates/sqlbuild-config/src/models.rs new file mode 100644 index 0000000000..8e84b2c66c --- /dev/null +++ b/crates/sqlbuild-config/src/models.rs @@ -0,0 +1,101 @@ +//! Configuration values with the value domain of Python's `tomllib` and PyYAML `safe_load`. + +use crate::constants::{ + FEBRUARY, MAX_HOUR, MAX_MICROSECOND, MAX_MINUTE, MAX_SECOND, MAX_YEAR, MIN_YEAR, + MONTHS_PER_YEAR, +}; + +/// A calendar date, Python's `datetime.date`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigDate { + pub year: u16, + pub month: u8, + pub day: u8, +} + +/// A time of day with microseconds, Python's naive `datetime.time`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigTime { + pub hour: u8, + pub minute: u8, + pub second: u8, + pub microsecond: u32, +} + +/// A date and time, Python's `datetime.datetime`, with its UTC offset in seconds when aware. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigDateTime { + pub date: ConfigDate, + pub time: ConfigTime, + pub utc_offset_seconds: Option, +} + +impl ConfigDate { + /// The date, or `None` where Python's `datetime.date` constructor raises. + pub fn checked(year: i64, month: i64, day: i64) -> Option { + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let month_days = match month { + FEBRUARY => 28 + i64::from(leap), + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + let valid = (MIN_YEAR..=MAX_YEAR).contains(&year) + && (1..=MONTHS_PER_YEAR).contains(&month) + && (1..=month_days).contains(&day); + valid.then_some(Self { + year: year as u16, + month: month as u8, + day: day as u8, + }) + } +} + +impl ConfigTime { + /// The time, or `None` where Python's `datetime.time` constructor raises. + pub fn checked(hour: i64, minute: i64, second: i64, microsecond: i64) -> Option { + let valid = (0..=MAX_HOUR).contains(&hour) + && (0..=MAX_MINUTE).contains(&minute) + && (0..=MAX_SECOND).contains(&second) + && (0..=MAX_MICROSECOND).contains(µsecond); + valid.then_some(Self { + hour: hour as u8, + minute: minute as u8, + second: second as u8, + microsecond: microsecond as u32, + }) + } +} + +/// One loaded configuration value; mappings keep document order and Python key equality. +#[derive(Clone, Debug, PartialEq)] +pub enum ConfigValue { + Null, + Bool(bool), + Integer(i64), + BigInteger(String), + Float(f64), + String(String), + Date(ConfigDate), + DateTime(ConfigDateTime), + Time(ConfigTime), + List(Vec), + Map(Vec<(ConfigValue, ConfigValue)>), +} + +impl ConfigValue { + /// The entries of a mapping, or `None` for any other value. + pub fn as_map(&self) -> Option<&[(ConfigValue, ConfigValue)]> { + match self { + Self::Map(entries) => Some(entries), + _ => None, + } + } + + /// The value of a string key in a mapping, as Python's `mapping.get(key)` finds it. + pub fn get(&self, key: &str) -> Option<&ConfigValue> { + self.as_map()? + .iter() + .find(|(candidate, _)| matches!(candidate, Self::String(text) if text == key)) + .map(|(_, value)| value) + } +} diff --git a/crates/sqlbuild-config/src/project/_helpers/fields.rs b/crates/sqlbuild-config/src/project/_helpers/fields.rs new file mode 100644 index 0000000000..af0621fe3c --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/fields.rs @@ -0,0 +1,92 @@ +//! Typed reads of configuration fields with the Python loader's defaults and type checks. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; + +fn invalid(field: &str, expected: &str) -> ConfigError { + ConfigError::new( + ConfigErrorKind::InvalidField, + format!("{field} must be {expected}"), + ) +} + +/// A section as a mapping; a missing section is empty, as in `_coerce_mapping`. +pub(crate) fn section<'value>( + table: &'value ConfigValue, + key: &str, +) -> Result<&'value [(ConfigValue, ConfigValue)], ConfigError> { + match table.get(key) { + None | Some(ConfigValue::Null) => Ok(&[]), + Some(value) => value.as_map().ok_or_else(|| invalid(key, "a mapping")), + } +} + +/// A non-empty string, stripped, as in `_require_str`. +pub(crate) fn required_text(table: &ConfigValue, key: &str) -> Result { + optional_text(table, key)?.ok_or_else(|| invalid(key, "a non-empty string")) +} + +/// An optional non-empty string, stripped, as in `_optional_str`. +pub(crate) fn optional_text(table: &ConfigValue, key: &str) -> Result, ConfigError> { + match table.get(key) { + None | Some(ConfigValue::Null) => Ok(None), + Some(ConfigValue::String(text)) if !text.trim().is_empty() => { + Ok(Some(text.trim().to_owned())) + } + Some(_) => Err(invalid(key, "a non-empty string")), + } +} + +/// An optional boolean with a default, as in `_optional_bool`. +pub(crate) fn flag( + entries: &[(ConfigValue, ConfigValue)], + key: &str, + default: bool, +) -> Result { + match lookup(entries, key) { + None | Some(ConfigValue::Null) => Ok(default), + Some(ConfigValue::Bool(value)) => Ok(*value), + Some(_) => Err(invalid(key, "a boolean")), + } +} + +/// The value of a string key among mapping entries. +pub(crate) fn lookup<'value>( + entries: &'value [(ConfigValue, ConfigValue)], + key: &str, +) -> Option<&'value ConfigValue> { + entries + .iter() + .find(|(candidate, _)| matches!(candidate, ConfigValue::String(text) if text == key)) + .map(|(_, value)| value) +} + +/// The text of a mapping key; TOML keys are always strings. +pub(crate) fn key_text(key: &ConfigValue) -> Result { + match key { + ConfigValue::String(text) => Ok(text.clone()), + _ => Err(invalid("a key", "a string")), + } +} + +/// String values, as in `_load_string_mapping`. +pub(crate) fn text_mapping( + table: &ConfigValue, + key: &str, +) -> Result, ConfigError> { + section(table, key)? + .iter() + .map(|(name, value)| match value { + ConfigValue::String(text) => Ok((key_text(name)?, text.clone())), + _ => Err(invalid(&format!("{key}.{}", key_text(name)?), "a string")), + }) + .collect() +} + +/// The keys of a section, in document order. +pub(crate) fn section_keys(table: &ConfigValue, key: &str) -> Result, ConfigError> { + section(table, key)? + .iter() + .map(|(name, _)| key_text(name)) + .collect() +} diff --git a/crates/sqlbuild-config/src/project/_helpers/files.rs b/crates/sqlbuild-config/src/project/_helpers/files.rs new file mode 100644 index 0000000000..c34d90dd4e --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/files.rs @@ -0,0 +1,35 @@ +//! Locate and load configuration files as the Python loader does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::toml::main::load_toml::load_toml; +use std::path::Path; + +/// Reject a legacy YAML file that has no TOML replacement, as the Python loader does. +pub(crate) fn reject_legacy( + directory: &Path, + filename: &str, + legacy_filename: &str, +) -> Result<(), ConfigError> { + if !directory.join(filename).exists() && directory.join(legacy_filename).exists() { + return Err(ConfigError::new( + ConfigErrorKind::InvalidField, + format!("{legacy_filename} is no longer supported; rename it to {filename}"), + )); + } + Ok(()) +} + +/// Read and parse one TOML file the way `tomllib.load` does. +pub(crate) fn load_toml_file(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|error| { + ConfigError::new( + ConfigErrorKind::Missing, + format!("{}: {error}", path.display()), + ) + })?; + let text = String::from_utf8(bytes).map_err(|error| { + ConfigError::new(ConfigErrorKind::Syntax, format!("invalid UTF-8: {error}")) + })?; + load_toml(&text) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/mod.rs b/crates/sqlbuild-config/src/project/_helpers/mod.rs new file mode 100644 index 0000000000..b21adf99f8 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod fields; +pub(crate) mod files; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs new file mode 100644 index 0000000000..850458ae50 --- /dev/null +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -0,0 +1,9 @@ +//! Project configuration file names and keys. + +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation"; +pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs new file mode 100644 index 0000000000..846d5991cb --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -0,0 +1,2 @@ +pub mod read_local_config; +pub mod read_project_config; diff --git a/crates/sqlbuild-config/src/project/main/read_local_config.rs b/crates/sqlbuild-config/src/project/main/read_local_config.rs new file mode 100644 index 0000000000..70d30f2e34 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/read_local_config.rs @@ -0,0 +1,40 @@ +//! Read the discovery fields of `sqlbuild_local.toml`. + +use crate::errors::ConfigError; +use crate::project::_helpers::fields::{ + flag, lookup, optional_text, section, section_keys, text_mapping, +}; +use crate::project::_helpers::files::{load_toml_file, reject_legacy}; +use crate::project::constants::{ + LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, LOCAL_CONFIG_FILENAME, + SQL_ANALYSIS_KEY, +}; +use crate::project::models::LocalConfigFile; +use std::path::Path; + +/// Read `sqlbuild_local.toml` in `project_dir`, or the defaults when it does not exist. +pub fn read_local_config(project_dir: &Path) -> Result { + reject_legacy( + project_dir, + LOCAL_CONFIG_FILENAME, + LEGACY_LOCAL_CONFIG_FILENAME, + )?; + let path = project_dir.join(LOCAL_CONFIG_FILENAME); + if !path.exists() { + return Ok(LocalConfigFile::default()); + } + let table = load_toml_file(&path)?; + let settings = section(&table, "settings")?; + let analysis_key = [SQL_ANALYSIS_KEY, LEGACY_SQL_VALIDATION_KEY] + .into_iter() + .find(|key| lookup(settings, key).is_some()); + Ok(LocalConfigFile { + target: optional_text(&table, "target")?, + adapter: optional_text(&table, "adapter")?, + sql_analysis: analysis_key + .map(|key| flag(settings, key, true)) + .transpose()?, + vars: text_mapping(&table, "vars")?, + target_names: section_keys(&table, "targets")?, + }) +} diff --git a/crates/sqlbuild-config/src/project/main/read_project_config.rs b/crates/sqlbuild-config/src/project/main/read_project_config.rs new file mode 100644 index 0000000000..1c345a5b16 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/read_project_config.rs @@ -0,0 +1,76 @@ +//! Read the discovery fields of `sqlbuild_project.toml`. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::project::_helpers::fields::{ + flag, key_text, lookup, optional_text, required_text, section, section_keys, text_mapping, +}; +use crate::project::_helpers::files::{load_toml_file, reject_legacy}; +use crate::project::constants::{ + LEGACY_PROJECT_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, PROJECT_CONFIG_FILENAME, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, +}; +use crate::project::models::{ProjectConfigFile, ProjectSettings}; +use std::path::Path; + +fn settings(table: &ConfigValue) -> Result { + let entries = section(table, "settings")?; + let current = lookup(entries, SQL_ANALYSIS_KEY); + let legacy = lookup(entries, LEGACY_SQL_VALIDATION_KEY); + if current.is_some() && legacy.is_some() && current != legacy { + return Err(ConfigError::new( + ConfigErrorKind::InvalidField, + "settings.sql_analysis conflicts with legacy settings.sql_validation", + )); + } + let analysis_key = if current.is_some() { + SQL_ANALYSIS_KEY + } else { + LEGACY_SQL_VALIDATION_KEY + }; + Ok(ProjectSettings { + sql_analysis: flag(entries, analysis_key, true)?, + require_sql_analysis: flag(entries, REQUIRE_SQL_ANALYSIS_KEY, false)?, + }) +} + +fn path_defaults(table: &ConfigValue) -> Result, ConfigError> { + section(table, "path_defaults")? + .iter() + .map(|(key, value)| { + let name = key_text(key)?; + match value { + ConfigValue::Map(_) => Ok((name, value.clone())), + _ => Err(ConfigError::new( + ConfigErrorKind::InvalidField, + format!("path_defaults['{name}'] must be a mapping"), + )), + } + }) + .collect() +} + +/// Read `sqlbuild_project.toml` in `project_dir`; Python's loader remains the full validator. +pub fn read_project_config(project_dir: &Path) -> Result { + reject_legacy( + project_dir, + PROJECT_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, + )?; + let table = load_toml_file(&project_dir.join(PROJECT_CONFIG_FILENAME))?; + Ok(ProjectConfigFile { + name: required_text(&table, "name")?, + adapter: required_text(&table, "adapter")?, + default_target: optional_text(&table, "default_target")?, + settings: settings(&table)?, + enforce_placement: flag(section(&table, "scopes")?, "enforce_placement", true)?, + enforce_explicit_references: flag( + section(&table, "references")?, + "enforce_explicit", + true, + )?, + vars: text_mapping(&table, "vars")?, + path_defaults: path_defaults(&table)?, + target_names: section_keys(&table, "targets")?, + }) +} diff --git a/crates/sqlbuild-config/src/project/mod.rs b/crates/sqlbuild-config/src/project/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-config/src/project/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs new file mode 100644 index 0000000000..5cdcb3fff1 --- /dev/null +++ b/crates/sqlbuild-config/src/project/models.rs @@ -0,0 +1,34 @@ +//! The project configuration fields native discovery reads. + +use crate::models::ConfigValue; + +/// Shared settings discovery reads from `[settings]`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectSettings { + pub sql_analysis: bool, + pub require_sql_analysis: bool, +} + +/// The discovery fields of `sqlbuild_project.toml`, with Python's defaults and stripping. +#[derive(Clone, Debug, PartialEq)] +pub struct ProjectConfigFile { + pub name: String, + pub adapter: String, + pub default_target: Option, + pub settings: ProjectSettings, + pub enforce_placement: bool, + pub enforce_explicit_references: bool, + pub vars: Vec<(String, String)>, + pub path_defaults: Vec<(String, ConfigValue)>, + pub target_names: Vec, +} + +/// The discovery fields of `sqlbuild_local.toml`; an absent file reads as the default. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LocalConfigFile { + pub target: Option, + pub adapter: Option, + pub sql_analysis: Option, + pub vars: Vec<(String, String)>, + pub target_names: Vec, +} diff --git a/crates/sqlbuild-config/src/project/tests.rs b/crates/sqlbuild-config/src/project/tests.rs new file mode 100644 index 0000000000..efd9060dc5 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_read_config.rs"] +mod read_config; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/project/tests/helpers.rs b/crates/sqlbuild-config/src/project/tests/helpers.rs new file mode 100644 index 0000000000..b7bb9560e6 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/helpers.rs @@ -0,0 +1,50 @@ +use crate::models::ConfigValue; +use crate::project::models::{ProjectConfigFile, ProjectSettings}; +use tempfile::TempDir; + +/// A project directory containing `files`. +pub(super) fn project_directory(files: &[(&str, &str)]) -> TempDir { + let directory = TempDir::new().expect("temporary directory"); + for (name, contents) in files { + std::fs::write(directory.path().join(name), contents).expect("write project file"); + } + directory +} + +pub(super) fn minimal_project() -> ProjectConfigFile { + ProjectConfigFile { + name: "orders".to_owned(), + adapter: "duckdb".to_owned(), + default_target: None, + settings: ProjectSettings { + sql_analysis: true, + require_sql_analysis: false, + }, + enforce_placement: true, + enforce_explicit_references: true, + vars: vec![], + path_defaults: vec![], + target_names: vec![], + } +} + +pub(super) fn configured_project() -> ProjectConfigFile { + ProjectConfigFile { + default_target: Some("dev".to_owned()), + settings: ProjectSettings { + sql_analysis: false, + require_sql_analysis: true, + }, + enforce_placement: false, + vars: vec![("region".to_owned(), "eu".to_owned())], + path_defaults: vec![( + "staging".to_owned(), + ConfigValue::Map(vec![( + ConfigValue::String("materialized".to_owned()), + ConfigValue::String("view".to_owned()), + )]), + )], + target_names: vec!["dev".to_owned(), "prod".to_owned()], + ..minimal_project() + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_read_config.rs b/crates/sqlbuild-config/src/project/tests/test_read_config.rs new file mode 100644 index 0000000000..5d8534ec76 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_read_config.rs @@ -0,0 +1,125 @@ +use crate::errors::ConfigErrorKind; +use crate::project::main::read_local_config::read_local_config; +use crate::project::main::read_project_config::read_project_config; +use crate::project::models::{LocalConfigFile, ProjectSettings}; +use crate::project::tests::helpers::{configured_project, minimal_project, project_directory}; +use crate::project::tests::test_types::{LocalConfigTestCase, ProjectConfigTestCase}; + +const PROJECT: &str = "sqlbuild_project.toml"; +const LOCAL: &str = "sqlbuild_local.toml"; + +#[test] +fn given_project_files_when_reading_then_discovery_fields_match_python_loader() { + let test_cases = [ + ProjectConfigTestCase { + description: "defaults apply and strings are stripped", + files: &[(PROJECT, "name = ' orders '\nadapter = \"duckdb\"\n")], + expected_config: Ok(minimal_project()), + }, + ProjectConfigTestCase { + description: "discovery sections are read in document order", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\ndefault_target = 'dev'\n\ + [settings]\nsql_analysis = false\nrequire_sql_analysis = true\n\ + [scopes]\nenforce_placement = false\n[vars]\nregion = 'eu'\n\ + [path_defaults.staging]\nmaterialized = 'view'\n\ + [targets.dev]\n[targets.prod]\n", + )], + expected_config: Ok(configured_project()), + }, + ProjectConfigTestCase { + description: "legacy sql_validation sets SQL analysis", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_validation = false\n", + )], + expected_config: Ok(crate::project::models::ProjectConfigFile { + settings: ProjectSettings { + sql_analysis: false, + require_sql_analysis: false, + }, + ..minimal_project() + }), + }, + ProjectConfigTestCase { + description: "conflicting analysis keys are rejected", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_analysis = true\nsql_validation = false\n", + )], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "blank name is rejected", + files: &[(PROJECT, "name = ' '\nadapter = 'duckdb'\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "non-string var is rejected", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[vars]\nlimit = 3\n", + )], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "legacy YAML project file is rejected", + files: &[("sqlbuild_project.yml", "name: orders\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "missing project file", + files: &[], + expected_config: Err(ConfigErrorKind::Missing), + }, + ]; + for test_case in test_cases { + let directory = project_directory(test_case.files); + let actual = read_project_config(directory.path()).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_local_files_when_reading_then_overrides_match_python_loader() { + let test_cases = [ + LocalConfigTestCase { + description: "absent local file reads as defaults", + files: &[], + expected_config: Ok(LocalConfigFile::default()), + }, + LocalConfigTestCase { + description: "local overrides are read", + files: &[( + LOCAL, + "target = 'dev'\n[settings]\nsql_validation = false\n[vars]\nregion = 'us'\n[targets.dev]\n", + )], + expected_config: Ok(LocalConfigFile { + target: Some("dev".to_owned()), + adapter: None, + sql_analysis: Some(false), + vars: vec![("region".to_owned(), "us".to_owned())], + target_names: vec!["dev".to_owned()], + }), + }, + LocalConfigTestCase { + description: "legacy YAML local file is rejected", + files: &[("sqlbuild_local.yml", "target: dev\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ]; + for test_case in test_cases { + let directory = project_directory(test_case.files); + let actual = read_local_config(directory.path()).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_config, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs new file mode 100644 index 0000000000..f234c337c6 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::project::models::{LocalConfigFile, ProjectConfigFile}; + +pub(super) struct ProjectConfigTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_config: Result, +} + +pub(super) struct LocalConfigTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_config: Result, +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/conversion.rs b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs new file mode 100644 index 0000000000..5486a937ad --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs @@ -0,0 +1,171 @@ +//! Convert a parsed TOML document into the values `tomllib` returns. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use toml_edit::{Array, Datetime, Document, InlineTable, Item, Key, Offset, Table, Value}; + +const NANOSECONDS_PER_MICROSECOND: u32 = 1_000; +/// No key inside: such values sort by their own key alone. +const NO_POSITION: usize = usize::MAX; +/// The deepest value nesting converted natively; the version check keeps documents well within it. +const MAX_DEPTH: usize = 512; +const SECONDS_PER_MINUTE: i32 = 60; + +fn invalid_datetime(message: &str) -> ConfigError { + ConfigError::new( + ConfigErrorKind::Syntax, + format!("invalid date or datetime: {message}"), + ) +} + +fn date(value: toml_edit::Date) -> Result { + ConfigDate::checked( + i64::from(value.year), + i64::from(value.month), + i64::from(value.day), + ) + .ok_or_else(|| invalid_datetime("the date does not exist")) +} + +fn time(value: toml_edit::Time) -> Result { + let second = value + .second + .ok_or_else(|| invalid_datetime("seconds are required before TOML 1.1"))?; + ConfigTime::checked( + i64::from(value.hour), + i64::from(value.minute), + i64::from(second), + i64::from(value.nanosecond.unwrap_or_default() / NANOSECONDS_PER_MICROSECOND), + ) + .ok_or_else(|| invalid_datetime("the time does not exist")) +} + +fn datetime(value: &Datetime) -> Result { + let offset = value.offset.map(|offset| match offset { + Offset::Z => 0, + Offset::Custom { minutes } => i32::from(minutes) * SECONDS_PER_MINUTE, + }); + match (value.date, value.time) { + (Some(day), Some(clock)) => Ok(ConfigValue::DateTime(ConfigDateTime { + date: date(day)?, + time: time(clock)?, + utc_offset_seconds: offset, + })), + (Some(day), None) => date(day).map(ConfigValue::Date), + (None, Some(clock)) => time(clock).map(ConfigValue::Time), + (None, None) => Err(invalid_datetime("empty datetime")), + } +} + +/// A converted value and the earliest source offset of a key inside it, which orders tables. +type Positioned = (ConfigValue, usize); + +fn too_deep() -> ConfigError { + ConfigError::new( + ConfigErrorKind::Unsupported, + "TOML values nest deeper than the native limit", + ) +} + +fn key_position(key: Option<&Key>) -> usize { + key.and_then(Key::span) + .map_or(NO_POSITION, |span| span.start) +} + +/// A dictionary in `tomllib`'s order: keys in the order their first statement created them. +fn ordered_map(mut entries: Vec<(usize, String, ConfigValue)>) -> Positioned { + entries.sort_by_key(|(position, _, _)| *position); + let first = entries + .first() + .map_or(NO_POSITION, |(position, _, _)| *position); + let converted: Vec<(ConfigValue, ConfigValue)> = entries + .into_iter() + .map(|(_, key, value)| (ConfigValue::String(key), value)) + .collect(); + (ConfigValue::Map(converted), first) +} + +fn list(items: Vec) -> Positioned { + let first = items + .iter() + .map(|(_, position)| *position) + .min() + .unwrap_or(NO_POSITION); + let values: Vec = items.into_iter().map(|(value, _)| value).collect(); + (ConfigValue::List(values), first) +} + +fn array(values: &Array, depth: usize) -> Result { + values + .iter() + .map(|item| value(item, depth + 1)) + .collect::, _>>() + .map(list) +} + +fn inline_table(entries: &InlineTable, depth: usize) -> Result { + let mut converted: Vec<(usize, String, ConfigValue)> = Vec::new(); + for (key, item) in entries.iter() { + let (converted_value, inner) = value(item, depth + 1)?; + let position = key_position(entries.key(key)).min(inner); + converted.push((position, key.to_owned(), converted_value)); + } + Ok(ordered_map(converted)) +} + +fn value(item: &Value, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(too_deep()); + } + let scalar = |converted: ConfigValue| (converted, NO_POSITION); + match item { + Value::String(text) => Ok(scalar(ConfigValue::String(text.value().clone()))), + Value::Integer(number) => Ok(scalar(ConfigValue::Integer(*number.value()))), + Value::Float(number) => Ok(scalar(ConfigValue::Float(*number.value()))), + Value::Boolean(flag) => Ok(scalar(ConfigValue::Bool(*flag.value()))), + Value::Datetime(moment) => datetime(moment.value()).map(scalar), + Value::Array(values) => array(values, depth), + Value::InlineTable(entries) => inline_table(entries, depth), + } +} + +fn item(entry: &Item, depth: usize) -> Result, ConfigError> { + match entry { + Item::None => Ok(None), + Item::Value(scalar) => value(scalar, depth).map(Some), + Item::Table(entries) => table(entries, depth).map(Some), + Item::ArrayOfTables(tables) => tables + .iter() + .map(|entries| table(entries, depth + 1)) + .collect::, _>>() + .map(|tables| Some(list(tables))), + } +} + +fn table(entries: &Table, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(too_deep()); + } + let mut converted: Vec<(usize, String, ConfigValue)> = Vec::new(); + for (key, entry) in entries.iter() { + if let Some((converted_value, inner)) = item(entry, depth + 1)? { + let position = key_position(entries.key(key)).min(inner); + converted.push((position, key.to_owned(), converted_value)); + } + } + Ok(ordered_map(converted)) +} + +/// Parse `text` and convert its root table. +pub(crate) fn parse_document(text: &str) -> Result { + let document = Document::parse(text).map_err(|error| { + let offset = error.span().map_or(0, |span| span.start); + ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.message().to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + } + })?; + table(document.as_table(), 0).map(|(converted, _)| converted) +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/mod.rs b/crates/sqlbuild-config/src/toml/_helpers/mod.rs new file mode 100644 index 0000000000..cd76bd6968 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod conversion; +pub(crate) mod namespaces; +pub(crate) mod version; diff --git a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs new file mode 100644 index 0000000000..9961ccba4f --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs @@ -0,0 +1,344 @@ +//! `tomllib`'s namespace rules: which tables, dotted keys and arrays of tables may be (re)opened. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use std::collections::HashMap; +use toml_parser::decoder::Encoding; +use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; +use toml_parser::{ErrorSink, ParseError, Raw, Source, Span}; + +/// `Flags.FROZEN`: a value written with `=`, which later statements cannot extend. +const FROZEN: u8 = 1; +/// `Flags.EXPLICIT_NEST`: a table declared by a header or opened by a dotted key. +const EXPLICIT_NEST: u8 = 2; +/// The container depth the version check already enforces. +const MAX_CONTAINER_DEPTH: u32 = 80; + +/// One key of `tomllib`'s `Flags` tree. +#[derive(Default)] +struct FlagNode { + flags: u8, + recursive: u8, + nested: HashMap, +} + +/// `tomllib`'s `Flags`, keyed by table names without array indices. +#[derive(Default)] +struct Flags { + root: HashMap, + pending: Vec>, +} + +impl Flags { + fn is(&self, key: &[String], flag: u8) -> bool { + let Some((stem, parent)) = key.split_last() else { + return false; + }; + let mut container = &self.root; + for name in parent { + let Some(inner) = container.get(name) else { + return false; + }; + if inner.recursive & flag != 0 { + return true; + } + container = &inner.nested; + } + container + .get(stem) + .is_some_and(|node| (node.flags | node.recursive) & flag != 0) + } + + fn set(&mut self, key: &[String], flag: u8, recursive: bool) { + let Some((stem, parent)) = key.split_last() else { + return; + }; + let mut container = &mut self.root; + for name in parent { + container = &mut container.entry(name.clone()).or_default().nested; + } + let node = container.entry(stem.clone()).or_default(); + match recursive { + true => node.recursive |= flag, + false => node.flags |= flag, + } + } + + fn unset_all(&mut self, key: &[String]) { + let Some((stem, parent)) = key.split_last() else { + return; + }; + let mut container = &mut self.root; + for name in parent { + let Some(inner) = container.get_mut(name) else { + return; + }; + container = &mut inner.nested; + } + container.remove(stem); + } + + fn finalize_pending(&mut self) { + for key in std::mem::take(&mut self.pending) { + self.set(&key, EXPLICIT_NEST, false); + } + } +} + +/// What `tomllib`'s `NestedDict` holds behind one key. +enum Entry { + Table(HashMap), + Tables(Vec>), + Value, +} + +/// `tomllib`'s `NestedDict`: the tables and values created so far. +#[derive(Default)] +struct NestedDict { + root: HashMap, +} + +impl NestedDict { + /// `get_or_create_nest`; `None` where `tomllib` raises "There is no nest behind this key". + fn nest(&mut self, key: &[String], access_lists: bool) -> Option<&mut HashMap> { + let mut container = &mut self.root; + for name in key { + let entry = container + .entry(name.clone()) + .or_insert_with(|| Entry::Table(HashMap::new())); + container = match entry { + Entry::Table(table) => table, + Entry::Tables(tables) if access_lists => tables.last_mut()?, + _ => return None, + }; + } + Some(container) + } +} + +/// The keys and values of one inline table, checked as `tomllib`'s `parse_inline_table` does. +#[derive(Default)] +struct InlineTable { + data: NestedDict, + flags: Flags, + key: Vec, +} + +impl InlineTable { + fn insert(&mut self, container_value: bool) -> bool { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return false; + }; + if self.flags.is(&key, FROZEN) { + return false; + } + let Some(table) = self.data.nest(parent, false) else { + return false; + }; + if table.contains_key(stem) { + return false; + } + table.insert(stem.clone(), Entry::Value); + if container_value { + self.flags.set(&key, FROZEN, true); + } + true + } +} + +/// An open array or inline table inside a value. +enum Frame { + Array, + InlineTable(InlineTable), +} + +/// Replays `tomllib.loads`' bookkeeping over parser events and records the first rejection. +struct NamespaceReceiver<'text> { + source: Source<'text>, + data: NestedDict, + flags: Flags, + header: Vec, + key: Vec, + frames: Vec, + rejected: Option<(usize, &'static str)>, +} + +impl NamespaceReceiver<'_> { + fn reject(&mut self, span: Span, reason: &'static str) { + self.rejected.get_or_insert((span.start(), reason)); + } + + fn decode(&self, span: Span, encoding: Option) -> String { + let raw = self.source.get(span).map_or("", |raw| raw.as_str()); + let mut decoded = String::new(); + let mut ignored: Vec = Vec::new(); + Raw::new_unchecked(raw, encoding, span).decode_key(&mut decoded, &mut ignored); + decoded + } + + /// `key_value_rule` for a key-value pair under the current header. + fn key_value(&mut self, span: Span, container_value: bool) { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return; + }; + for length in 1..key.len() { + let container: Vec = [&self.header[..], &key[..length]].concat(); + if self.flags.is(&container, EXPLICIT_NEST) { + return self.reject(span, "Cannot redefine namespace"); + } + self.flags.pending.push(container); + } + let parent_key: Vec = [&self.header[..], parent].concat(); + if self.flags.is(&parent_key, FROZEN) { + return self.reject(span, "Cannot mutate immutable namespace"); + } + let Some(table) = self.data.nest(&parent_key, true) else { + return self.reject(span, "Cannot overwrite a value"); + }; + if table.contains_key(stem) { + return self.reject(span, "Cannot overwrite a value"); + } + table.insert(stem.clone(), Entry::Value); + if container_value { + let full_key: Vec = [&self.header[..], &key[..]].concat(); + self.flags.set(&full_key, FROZEN, true); + } + } + + /// `create_dict_rule` for a `[table]` header. + fn table(&mut self, span: Span) { + let key = std::mem::take(&mut self.key); + if self.flags.is(&key, EXPLICIT_NEST) || self.flags.is(&key, FROZEN) { + return self.reject(span, "Cannot declare a table twice"); + } + self.flags.set(&key, EXPLICIT_NEST, false); + if self.data.nest(&key, true).is_none() { + return self.reject(span, "Cannot overwrite a value"); + } + self.header = key; + } + + /// `create_list_rule` for an `[[array of tables]]` header. + fn array_of_tables(&mut self, span: Span) { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return; + }; + if self.flags.is(&key, FROZEN) { + return self.reject(span, "Cannot mutate immutable namespace"); + } + self.flags.unset_all(&key); + self.flags.set(&key, EXPLICIT_NEST, false); + let Some(table) = self.data.nest(parent, true) else { + return self.reject(span, "Cannot overwrite a value"); + }; + match table.get_mut(stem) { + None => { + table.insert(stem.clone(), Entry::Tables(vec![HashMap::new()])); + } + Some(Entry::Tables(tables)) => tables.push(HashMap::new()), + Some(_) => return self.reject(span, "Cannot overwrite a value"), + } + self.header = key; + } + + /// A complete value: a scalar, or an array or inline table that just closed. + fn value(&mut self, span: Span, container_value: bool) { + match self.frames.last_mut() { + None => self.key_value(span, container_value), + Some(Frame::Array) => {} + Some(Frame::InlineTable(table)) => { + if !table.insert(container_value) { + self.reject(span, "Cannot redefine an inline table key"); + } + } + } + } + + fn open_header(&mut self) { + self.flags.finalize_pending(); + self.key.clear(); + } +} + +impl EventReceiver for NamespaceReceiver<'_> { + fn std_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.open_header(); + } + + fn std_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.table(span); + } + + fn array_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.open_header(); + } + + fn array_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.array_of_tables(span); + } + + fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.frames.push(Frame::InlineTable(InlineTable::default())); + true + } + + fn inline_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.frames.pop(); + self.value(span, true); + } + + fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.frames.push(Frame::Array); + true + } + + fn array_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.frames.pop(); + self.value(span, true); + } + + fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + let segment = self.decode(span, encoding); + match self.frames.last_mut() { + Some(Frame::InlineTable(table)) => table.key.push(segment), + Some(Frame::Array) => {} + None => self.key.push(segment), + } + } + + fn scalar(&mut self, span: Span, _encoding: Option, _error: &mut dyn ErrorSink) { + self.value(span, false); + } +} + +/// Reject what `tomllib` rejects as a redefined or immutable namespace. +pub(crate) fn reject_namespace_conflicts(text: &str) -> Result<(), ConfigError> { + let source = Source::new(text); + let tokens = source.lex().into_vec(); + let mut receiver = NamespaceReceiver { + source, + data: NestedDict::default(), + flags: Flags::default(), + header: Vec::new(), + key: Vec::new(), + frames: Vec::new(), + rejected: None, + }; + let mut errors: Vec = Vec::new(); + parse_document( + &tokens, + &mut RecursionGuard::new(&mut receiver, MAX_CONTAINER_DEPTH), + &mut errors, + ); + match receiver.rejected { + None => Ok(()), + Some((offset, reason)) => Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: reason.to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }), + } +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/version.rs b/crates/sqlbuild-config/src/toml/_helpers/version.rs new file mode 100644 index 0000000000..008be45309 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/version.rs @@ -0,0 +1,182 @@ +//! Find TOML 1.1 syntax, which the parser accepts and Python's TOML 1.0 `tomllib` rejects. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use toml_parser::decoder::Encoding; +use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; +use toml_parser::{ErrorSink, ParseError, Source, Span}; + +/// The container depth `toml_edit` accepts; deeper documents are left to Python. +const MAX_CONTAINER_DEPTH: u32 = 80; +/// The most dotted key segments read natively, which bounds the table nesting of a document. +const MAX_KEY_SEGMENTS: usize = 128; + +/// The containers open at the current event. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Container { + InlineTable, + Array, +} + +/// Records the first TOML 1.1-only construct. +struct VersionReceiver<'text> { + source: Source<'text>, + containers: Vec, + after_comma: bool, + key_segments: usize, + found: Option<(usize, &'static str)>, +} + +impl VersionReceiver<'_> { + fn in_inline_table(&self) -> bool { + self.containers.last() == Some(&Container::InlineTable) + } + + fn record(&mut self, span: Span, construct: &'static str) { + self.found.get_or_insert((span.start(), construct)); + } + + fn token(&mut self) { + self.after_comma = false; + } + + fn end_key(&mut self) { + self.token(); + self.key_segments = 0; + } + + fn line_break(&mut self, span: Span) { + if self.in_inline_table() { + self.record(span, "a newline or comment inside an inline table"); + } + } + + /// Escapes added by TOML 1.1: `\e` and `\xHH`. + fn check_escapes(&mut self, span: Span, encoding: Option) { + if !matches!( + encoding, + Some(Encoding::BasicString | Encoding::MlBasicString) + ) { + return; + } + let raw = self + .source + .get(span) + .map(|raw| raw.as_str()) + .unwrap_or_default(); + let mut escaped = false; + for character in raw.chars() { + if escaped && matches!(character, 'e' | 'x') { + self.record(span, "a \\e or \\x escape"); + } + escaped = !escaped && character == '\\'; + } + } +} + +impl EventReceiver for VersionReceiver<'_> { + fn std_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn array_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.end_key(); + self.containers.push(Container::InlineTable); + true + } + + fn inline_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + if self.after_comma { + self.record(span, "a trailing comma in an inline table"); + } + self.token(); + self.containers.pop(); + } + + fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.token(); + self.containers.push(Container::Array); + true + } + + fn array_close(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.token(); + self.containers.pop(); + } + + fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + self.token(); + self.key_segments += 1; + if self.key_segments > MAX_KEY_SEGMENTS { + self.record(span, "a key with more segments than the native limit"); + } + self.check_escapes(span, encoding); + } + + fn key_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.token(); + } + + fn key_val_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn scalar(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + self.token(); + self.check_escapes(span, encoding); + } + + fn value_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.key_segments = 0; + self.after_comma = self.in_inline_table(); + } + + fn comment(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.line_break(span); + } + + fn newline(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.key_segments = 0; + self.line_break(span); + } +} + +/// Reject TOML 1.1-only syntax, syntax errors, and nesting beyond the native limits. +pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { + let source = Source::new(text); + let tokens = source.lex().into_vec(); + let mut receiver = VersionReceiver { + source, + containers: Vec::new(), + after_comma: false, + key_segments: 0, + found: None, + }; + let mut errors: Vec = Vec::new(); + parse_document( + &tokens, + &mut RecursionGuard::new(&mut receiver, MAX_CONTAINER_DEPTH), + &mut errors, + ); + if let Some(error) = errors.first() { + let offset = error.unexpected().map_or(0, |span| span.start()); + return Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.description().to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }); + } + match receiver.found { + None => Ok(()), + Some((offset, construct)) => Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: format!("{construct} is TOML 1.1 syntax, which tomllib rejects"), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }), + } +} diff --git a/crates/sqlbuild-config/src/toml/main/load_toml.rs b/crates/sqlbuild-config/src/toml/main/load_toml.rs new file mode 100644 index 0000000000..9e7ab66184 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/load_toml.rs @@ -0,0 +1,19 @@ +//! Load a TOML document the way Python's `tomllib.loads` does. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; + +/// The table `tomllib.loads(text)` returns, rejecting TOML 1.1 syntax as `tomllib` does. +pub fn load_toml(text: &str) -> Result { + if text.starts_with(BYTE_ORDER_MARK) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "invalid statement: tomllib does not skip a byte order mark", + )); + } + let text = text.replace("\r\n", "\n"); + crate::toml::_helpers::version::reject_toml_1_1_syntax(&text)?; + crate::toml::_helpers::namespaces::reject_namespace_conflicts(&text)?; + crate::toml::_helpers::conversion::parse_document(&text) +} diff --git a/crates/sqlbuild-config/src/toml/main/mod.rs b/crates/sqlbuild-config/src/toml/main/mod.rs new file mode 100644 index 0000000000..04683b5110 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/mod.rs @@ -0,0 +1 @@ +pub mod load_toml; diff --git a/crates/sqlbuild-config/src/toml/mod.rs b/crates/sqlbuild-config/src/toml/mod.rs new file mode 100644 index 0000000000..668a17d0aa --- /dev/null +++ b/crates/sqlbuild-config/src/toml/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/toml/tests.rs b/crates/sqlbuild-config/src/toml/tests.rs new file mode 100644 index 0000000000..55b6a54c6d --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_toml.rs"] +mod load_toml; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/toml/tests/helpers.rs b/crates/sqlbuild-config/src/toml/tests/helpers.rs new file mode 100644 index 0000000000..d8cd6e7cdd --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/helpers.rs @@ -0,0 +1,11 @@ +use crate::models::ConfigValue; + +/// A table from string keys, in the given order. +pub(super) fn table(entries: Vec<(&str, ConfigValue)>) -> ConfigValue { + ConfigValue::Map( + entries + .into_iter() + .map(|(key, value)| (ConfigValue::String(key.to_owned()), value)) + .collect(), + ) +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs new file mode 100644 index 0000000000..078edffbd4 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs @@ -0,0 +1,330 @@ +use crate::errors::ConfigErrorKind; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::toml::main::load_toml::load_toml; +use crate::toml::tests::helpers::table; +use crate::toml::tests::test_types::{HostileTomlTestCase, LoadTomlTestCase}; + +#[test] +fn given_toml_documents_when_loading_then_values_match_tomllib() { + let test_cases = [ + LoadTomlTestCase { + description: "tables keep document order", + text: "z = 1\n[b]\nx = true\n[a]\ny = 'q'", + expected_value: Ok(ConfigValue::Map(vec![ + (ConfigValue::String("z".to_owned()), ConfigValue::Integer(1)), + ( + ConfigValue::String("b".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("x".to_owned()), + ConfigValue::Bool(true), + )]), + ), + ( + ConfigValue::String("a".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("y".to_owned()), + ConfigValue::String("q".to_owned()), + )]), + ), + ])), + }, + LoadTomlTestCase { + description: "implicit parent tables are created where first named", + text: "[a.b]\nx = 1\n[c]\n[a.d]\ny = 2", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("a".to_owned()), + ConfigValue::Map(vec![ + ( + ConfigValue::String("b".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("x".to_owned()), + ConfigValue::Integer(1), + )]), + ), + ( + ConfigValue::String("d".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("y".to_owned()), + ConfigValue::Integer(2), + )]), + ), + ]), + ), + ( + ConfigValue::String("c".to_owned()), + ConfigValue::Map(vec![]), + ), + ])), + }, + LoadTomlTestCase { + description: "dotted keys and arrays of tables", + text: "p.q = 1\n[[r]]\ns = 1.5\n[[r]]", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("p".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("q".to_owned()), + ConfigValue::Integer(1), + )]), + ), + ( + ConfigValue::String("r".to_owned()), + ConfigValue::List(vec![ + ConfigValue::Map(vec![( + ConfigValue::String("s".to_owned()), + ConfigValue::Float(1.5), + )]), + ConfigValue::Map(vec![]), + ]), + ), + ])), + }, + LoadTomlTestCase { + description: "offset datetime keeps its offset", + text: "t = 1979-05-27T00:32:00.999999999-07:00", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("t".to_owned()), + ConfigValue::DateTime(ConfigDateTime { + date: ConfigDate { + year: 1979, + month: 5, + day: 27, + }, + time: ConfigTime { + hour: 0, + minute: 32, + second: 0, + microsecond: 999_999, + }, + utc_offset_seconds: Some(-25_200), + }), + )])), + }, + LoadTomlTestCase { + description: "local date and time", + text: "d = 1979-05-27\nt = 07:32:00", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("d".to_owned()), + ConfigValue::Date(ConfigDate { + year: 1979, + month: 5, + day: 27, + }), + ), + ( + ConfigValue::String("t".to_owned()), + ConfigValue::Time(ConfigTime { + hour: 7, + minute: 32, + second: 0, + microsecond: 0, + }), + ), + ])), + }, + LoadTomlTestCase { + description: "escaped backslash before e is TOML 1.0", + text: "a = \"\\\\e\"", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("a".to_owned()), + ConfigValue::String("\\e".to_owned()), + )])), + }, + LoadTomlTestCase { + description: "multi-line array is TOML 1.0", + text: "a = [\n1,\n]", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("a".to_owned()), + ConfigValue::List(vec![ConfigValue::Integer(1)]), + )])), + }, + LoadTomlTestCase { + description: "trailing comma in an inline table is TOML 1.1", + text: "a = {x = 1,}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "newline in an inline table is TOML 1.1", + text: "a = {\nx = 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "escape e is TOML 1.1", + text: "a = \"\\e\"", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "escape x is TOML 1.1", + text: "a = \"\\x41\"", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "time without seconds is TOML 1.1", + text: "a = 07:32", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "leap second is rejected", + text: "a = 1990-12-31T23:59:60Z", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "impossible date is rejected", + text: "a = 2021-02-30", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "duplicate keys are rejected", + text: "a = 1\na = 2", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an implicit table declared later keeps the position where it was created", + text: "[p.a.config]\nx = 1\n[p.b]\ny = 2\n[p.a]\nz = 3", + expected_value: Ok(table(vec![( + "p", + table(vec![ + ( + "a", + table(vec![ + ("config", table(vec![("x", ConfigValue::Integer(1))])), + ("z", ConfigValue::Integer(3)), + ]), + ), + ("b", table(vec![("y", ConfigValue::Integer(2))])), + ]), + )])), + }, + LoadTomlTestCase { + description: "a dotted key creates its table before a later header", + text: "q.x = 1\n[r]\n[q.s]", + expected_value: Ok(table(vec![ + ( + "q", + table(vec![("x", ConfigValue::Integer(1)), ("s", table(vec![]))]), + ), + ("r", table(vec![])), + ])), + }, + LoadTomlTestCase { + description: "a dotted key cannot reach into an array of tables", + text: "[[a.b]]\n[a]\nb.c.d = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a two-segment dotted key cannot reach into an array of tables", + text: "[[a.b]]\n[a]\nb.c = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a dotted key cannot reopen a table declared by a header", + text: "[a.b]\n[a]\nb.c = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot redeclare a table created by a dotted key", + text: "a.b = 1\n[a]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot extend a table opened by dotted keys in a section", + text: "[a]\nb.c = 1\n[a.b]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot extend an inline table", + text: "a = {b = 1}\n[a.c]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an inline table cannot be extended by a dotted key", + text: "x = {a = {b = 1}, a.c = 2}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an array of tables cannot extend a static array", + text: "a = [1]\n[[a]]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "each array of tables element reopens its own sub-tables", + text: "[[t]]\n[t.c]\nx = 1\n[[t]]\n[t.c]\ny = 2", + expected_value: Ok(table(vec![( + "t", + ConfigValue::List(vec![ + table(vec![("c", table(vec![("x", ConfigValue::Integer(1))]))]), + table(vec![("c", table(vec![("y", ConfigValue::Integer(2))]))]), + ]), + )])), + }, + LoadTomlTestCase { + description: "dotted keys may extend the tables they created", + text: "a.b.c = 1\na.b.d = 2", + expected_value: Ok(table(vec![( + "a", + table(vec![( + "b", + table(vec![ + ("c", ConfigValue::Integer(1)), + ("d", ConfigValue::Integer(2)), + ]), + )]), + )])), + }, + LoadTomlTestCase { + description: "a byte order mark is rejected as tomllib rejects it", + text: "\u{feff}a = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "CRLF line endings are normalized inside multi-line strings", + text: "a = \"\"\"x\r\ny\"\"\"", + expected_value: Ok(table(vec![("a", ConfigValue::String("x\ny".to_owned()))])), + }, + ]; + for test_case in test_cases { + let actual = load_toml(test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hostile_toml_when_loading_then_native_reader_rejects_without_exhausting_the_stack() { + let test_cases = [ + HostileTomlTestCase { + description: "ten thousand nested arrays", + text: format!("a = {}{}", "[".repeat(10_000), "]".repeat(10_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "ten thousand nested inline tables", + text: format!("a = {}{}", "{b = ".repeat(10_000), "}".repeat(10_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "a dotted key with a hundred thousand segments", + text: format!("a{} = 1", ".a".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "a table header with a hundred thousand segments", + text: format!("[a{}]", ".a".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + ]; + for test_case in test_cases { + let actual = load_toml(&test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, + Err(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs new file mode 100644 index 0000000000..cb0c0f42cf --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; + +pub(super) struct LoadTomlTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, +} + +pub(super) struct HostileTomlTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_error: ConfigErrorKind, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 14f33f0798..bf813b9f9f 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -21,6 +21,7 @@ mimalloc.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-rules.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index 8b1ad68dbc..f16f61ab8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -3,13 +3,19 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use serde_json::Value; +use serde_json::{Value, json}; +use sqlbuild_config::errors::ConfigError; +use sqlbuild_config::models::{ConfigDate, ConfigTime, ConfigValue}; +use sqlbuild_config::project::main::read_local_config::read_local_config; +use sqlbuild_config::project::main::read_project_config::read_project_config; +use sqlbuild_config::toml::main::load_toml::load_toml; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{ JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, }; use sqlbuild_core::text::main::decode_python_text::decode_python_text; use sqlbuild_core::text::models::LineIndex; +use std::path::Path; fn oracle_error(message: impl std::fmt::Display) -> PyErr { PyValueError::new_err(message.to_string()) @@ -130,8 +136,93 @@ fn _oracle_text_positions( Ok((text, positions)) } +fn date_parts(date: ConfigDate) -> Value { + json!([date.year, date.month, date.day]) +} + +fn time_parts(time: ConfigTime) -> Value { + json!([time.hour, time.minute, time.second, time.microsecond]) +} + +fn float_canonical(value: f64) -> Value { + if value.is_nan() { + return json!(["float", "nan"]); + } + json!(["float", value.to_bits().to_string()]) +} + +/// The tagged canonical form the Python oracle tests build from `tomllib` and PyYAML values. +fn canonical(value: &ConfigValue) -> Value { + match value { + ConfigValue::Null => json!(["null"]), + ConfigValue::Bool(flag) => json!(["bool", flag]), + ConfigValue::Integer(number) => json!(["int", number.to_string()]), + ConfigValue::BigInteger(text) => json!(["int", text]), + ConfigValue::Float(number) => float_canonical(*number), + ConfigValue::String(text) => json!(["str", text]), + ConfigValue::Date(date) => json!(["date", date_parts(*date)]), + ConfigValue::DateTime(moment) => json!([ + "datetime", + date_parts(moment.date), + time_parts(moment.time), + moment.utc_offset_seconds + ]), + ConfigValue::Time(time) => json!(["time", time_parts(*time)]), + ConfigValue::List(items) => { + json!(["list", items.iter().map(canonical).collect::>()]) + } + ConfigValue::Map(entries) => json!([ + "map", + entries + .iter() + .map(|(key, item)| json!([canonical(key), canonical(item)])) + .collect::>() + ]), + } +} + +fn outcome(result: Result) -> String { + result + .unwrap_or_else(|error| json!({"error": format!("{:?}", error.kind)})) + .to_string() +} + +/// Load TOML natively and return its canonical form, or `{"error": kind}`. +#[pyfunction] +fn _oracle_toml_load(text: &str) -> String { + outcome(load_toml(text).map(|value| canonical(&value))) +} + +/// Read a project's discovery configuration natively, or return `{"error": kind}`. +#[pyfunction] +fn _oracle_project_config(project_dir: &str) -> String { + let directory = Path::new(project_dir); + outcome(read_project_config(directory).and_then(|project| { + let local = read_local_config(directory)?; + Ok(json!({ + "name": project.name, + "adapter": project.adapter, + "default_target": project.default_target, + "sql_analysis": project.settings.sql_analysis, + "require_sql_analysis": project.settings.require_sql_analysis, + "enforce_placement": project.enforce_placement, + "enforce_explicit_references": project.enforce_explicit_references, + "vars": project.vars, + "path_default_keys": project.path_defaults.iter().map(|(key, _)| key).collect::>(), + "target_names": project.target_names, + "local_target": local.target, + "local_adapter": local.adapter, + "local_sql_analysis": local.sql_analysis, + "local_vars": local.vars, + "local_target_names": local.target_names, + })) + })) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_toml_load, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_project_config, module)?)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index 89e4d7f529..de66a371f9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -305,6 +305,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ "sqlbuild-analysis", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-python", "sqlbuild-rules", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c98c22bfaf..add85bee8c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -55,3 +55,5 @@ def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] ) -> tuple[str, list[tuple[int, int, int] | None]]: ... +def _oracle_toml_load(text: str) -> str: ... +def _oracle_project_config(project_dir: str) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/discovery/__init__.py b/tests/integration/src/sqlbuild/compiler/discovery/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py new file mode 100644 index 0000000000..8967c0f188 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedDocumentOracleTestCase: + """Seeded generated documents loaded natively and by the Python library they mirror.""" + + description: str + generator: str + seed: int + case_count: int + expected_maximum_deferred: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class RepositoryFileOracleTestCase: + """Repository files of one kind loaded natively and by the Python loader.""" + + description: str + pattern: str + expected_minimum_files: int + expected_deferred: int = 0 + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class HostileDocumentOracleTestCase: + """A document built to exhaust a naive reader, which the native reader must defer quickly.""" + + description: str + document: str + expected_deferred: bool = True + expected_maximum_seconds: float = 1.0 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py new file mode 100644 index 0000000000..8a6581fb43 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -0,0 +1,346 @@ +"""Seeded TOML generators and canonical values for the native configuration oracles.""" + +from __future__ import annotations + +import datetime +import json +import math +import random +import struct +import tomllib +from collections.abc import Callable +from contextlib import suppress +from itertools import compress +from pathlib import Path +from typing import Any + +from sqlbuild import _native +from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text + +REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] +EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".venv", "node_modules", "target", ".git"}) +PYTHON_ERROR: dict[str, str] = {"error": "rejected"} +DEFERRED: dict[str, str] = {"error": "deferred to Python"} +UNCHECKED_PYTHON_OUTCOMES: tuple[dict[str, str], ...] = (PYTHON_ERROR,) +YAML_LINE_BREAKS: dict[int, None] = {0x85: None, 0x2028: None, 0x2029: None} +BYTE_ORDER_MARKS: tuple[str, ...] = ("",) * 19 + ("\ufeff",) + + +def yaml_text(*, rng: random.Random, max_length: int) -> str: + """Return random text without the line breaks only YAML 1.1 recognizes.""" + + return random_text(rng=rng, max_length=max_length).translate(YAML_LINE_BREAKS) + + +HOSTILE_TOML_DOCUMENTS: dict[str, Callable[[], str]] = { + "deep arrays": lambda: "a = " + "[" * 10_000 + "]" * 10_000, + "deep inline tables": lambda: "a = " + "{b = " * 10_000 + "}" * 10_000, + "long dotted key": lambda: "a" + ".a" * 100_000 + " = 1", + "long table header": lambda: "[a" + ".a" * 100_000 + "]", +} + + +def _toml_key(rng: random.Random, index: int) -> str: + bare: str = f"k{index}_{rng.choice(('a', 'B', '1', '-', '_'))}" + quoted: str = json.dumps(yaml_text(rng=rng, max_length=5) + str(index), ensure_ascii=False) + literal: str = f"'{rng.choice(('é', 'x y', 'dot.ted'))}{index}'" + return rng.choice((bare, bare, quoted, literal)) + + +def _toml_datetime(rng: random.Random) -> str: + date: str = f"{rng.randint(1, 9999):04d}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" + time: str = f"{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + fraction: str = rng.choice(("", f".{rng.randint(0, 10**9)}", ".5")) + offset: str = rng.choice( + ("", "Z", "z", f"{rng.choice('+-')}{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}") + ) + return rng.choice( + (date, f"{time}{fraction}", f"{date}T{time}{fraction}{offset}", f"{date} {time}{offset}") + ) + + +def _toml_float(rng: random.Random) -> str: + value: float = random_float(rng=rng) + exponent: str = f"{rng.randint(-99, 99)}.{rng.randint(0, 99)}e{rng.randint(-30, 30)}" + return rng.choice((repr(value), exponent, "1_000.5", "-0.0", "+inf", "-nan")) + + +TOML_ESCAPES: tuple[str, ...] = ( + "\\b", "\\t", "\\n", "\\f", "\\r", '\\"', "\\\\", "\\u00e9", "\\U0001F600", +) # fmt: skip +TOML_STRING_LITERALS: tuple[str, ...] = ( + "'C:\\path'", + "'a \"q\"'", + "'é'", + '"""\nline"""', + '"""\none\\\n two"""', + "'''\nraw \\n'''", + "'''\ntwo\nlines'''", +) +TOML_1_1_SYNTAX: tuple[str, ...] = ( + '"\\e"', + '"\\x41"', + "{x = 1,}", + "{\n x = 1\n}", + "{x = 1, # note\n}", + "07:32", + "1979-05-27T07:32Z", + "1979-05-27 07:32", +) +TOML_TABLE_NAMES: tuple[str, ...] = ("p", "p.q", "p.q.r", "p.s", "t", "t.u", '"p".v') +TOML_SCALARS: tuple[Callable[[random.Random], str], ...] = ( + lambda rng: rng.choice((*TOML_1_1_SYNTAX, str(2**64), str(-(2**80)))), + lambda rng: str(rng.randint(-(2**63), 2**63 - 1)), + lambda rng: rng.choice(("0x1F", "0o17", "0b101", "1_000", "+7", "-0")), + _toml_float, + lambda rng: rng.choice(("true", "false")), + lambda rng: json.dumps(yaml_text(rng=rng, max_length=10), ensure_ascii=False), + lambda rng: '"' + "".join(rng.choice(TOML_ESCAPES) for _ in range(rng.randint(0, 3))) + 'x"', + lambda rng: rng.choice(TOML_STRING_LITERALS), + _toml_datetime, +) + + +def _toml_inline(rng: random.Random, depth: int) -> str: + entries: str = ", ".join( + f"{_toml_key(rng, index)} = {random_toml_value(rng=rng, depth=depth - 1)}" + for index in range(rng.randint(0, 3)) + ) + return "{" + entries + "}" + + +def _toml_array(rng: random.Random, depth: int) -> str: + values: list[str] = [ + random_toml_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 3)) + ] + return "[" + rng.choice((", ", ",\n ")).join(values) + rng.choice(("", ",")) + "]" + + +TOML_GENERATORS: tuple[Callable[[random.Random, int], str], ...] = ( + *(lambda rng, depth, scalar=scalar: scalar(rng) for scalar in TOML_SCALARS), + _toml_inline, + _toml_array, +) + + +def random_toml_value(*, rng: random.Random, depth: int) -> str: + """Return the TOML text of a random value.""" + + available: int = len(TOML_SCALARS) + 2 * min(depth, 1) + return rng.choice(TOML_GENERATORS[:available])(rng, depth) + + +def _toml_pairs(rng: random.Random, prefix: str) -> str: + return "".join( + f"{prefix}{_toml_key(rng, index)} = {random_toml_value(rng=rng, depth=2)}" + f"{rng.choice(('', ' # note'))}\n" + for index in range(rng.randint(0, 4)) + ) + + +def toml_document(*, rng: random.Random) -> str: + """Return a TOML document with root keys, dotted keys, tables and arrays of tables. + + Table names repeat across sections, so implicit tables are later declared explicitly or + extended through dotted keys, as `tomllib` orders them by first creation. + """ + + sections: list[str] = [ + rng.choice( + ( + f"[table{index}]\n{_toml_pairs(rng, '')}", + f"[parent.child{index}]\n{_toml_pairs(rng, '')}", + f"[[rows]]\n{_toml_pairs(rng, '')}", + f"{_toml_pairs(rng, f'dotted{index}.')}", + f"[{rng.choice(TOML_TABLE_NAMES)}]\n{_toml_pairs(rng, '')}", + f"[[{rng.choice(TOML_TABLE_NAMES)}]]\n{_toml_pairs(rng, '')}", + f"{_toml_pairs(rng, rng.choice(TOML_TABLE_NAMES) + '.')}", + f"[late{index}.inner]\n{_toml_pairs(rng, '')}[side{index}]\n" + f"{_toml_pairs(rng, '')}[late{index}]\n{_toml_pairs(rng, '')}", + f"[[late{index}.rows]]\n{_toml_pairs(rng, '')}[late{index}]\n" + f"{_toml_pairs(rng, f'more{index}.')}", + ) + ) + for index in range(rng.randint(0, 6)) + ] + document: str = _toml_pairs(rng, "") + "\n".join(sections) + line_endings: str = rng.choice((document, document, document.replace("\n", "\r\n"))) + return rng.choice(BYTE_ORDER_MARKS) + line_endings + + +TOML_NAMESPACE_NAMES: tuple[str, ...] = ( + "a", "b", "a.b", "a.c", "b.a", "a.b.c", "a.b.c.d", "b.c.d", '"a".b', "a.'b'.c", +) # fmt: skip +TOML_NAMESPACE_VALUES: tuple[str, ...] = ( + "1", "{}", "{x = 1}", "{c = {d = 1}}", "[]", "[{x = 1}]", "{b.c = 1}", "[1, 2]", +) # fmt: skip +TOML_NAMESPACE_STATEMENTS: tuple[Callable[[random.Random], str], ...] = ( + lambda rng: f"[{rng.choice(TOML_NAMESPACE_NAMES)}]", + lambda rng: f"[[{rng.choice(TOML_NAMESPACE_NAMES)}]]", + lambda rng: f"{rng.choice(TOML_NAMESPACE_NAMES)} = {rng.choice(TOML_NAMESPACE_VALUES)}", + lambda rng: f"{rng.choice(TOML_NAMESPACE_NAMES)}.k{rng.randint(0, 2)} = 1", + lambda rng: ( + f"x = {{{rng.choice(TOML_NAMESPACE_NAMES)} = 1, {rng.choice(TOML_NAMESPACE_NAMES)} = 2}}" + ), +) + + +def toml_namespace_document(*, rng: random.Random) -> str: + """Return statements that reopen tables, arrays of tables and dotted keys in random orders.""" + + return "\n".join(rng.choice(TOML_NAMESPACE_STATEMENTS)(rng) for _ in range(rng.randint(1, 8))) + + +TOML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "toml": lambda rng: toml_document(rng=rng), + "toml namespaces": lambda rng: toml_namespace_document(rng=rng), +} + + +def _float_canonical(value: float) -> list[object]: + bits: str = str(struct.unpack(" int | None: + offset: datetime.timedelta | None = value.utcoffset() + return OFFSET_SECONDS[type(offset)](offset) + + +CANONICAL: dict[type, Callable[[Any], object]] = { + type(None): lambda value: ["null"], + bool: lambda value: ["bool", value], + int: lambda value: ["int", str(value)], + float: _float_canonical, + str: lambda value: ["str", value], + datetime.date: lambda value: ["date", [value.year, value.month, value.day]], + datetime.datetime: lambda value: [ + "datetime", + [value.year, value.month, value.day], + [value.hour, value.minute, value.second, value.microsecond], + _utc_offset(value), + ], + datetime.time: lambda value: [ + "time", + [value.hour, value.minute, value.second, value.microsecond], + ], + list: lambda value: ["list", [canonical(item) for item in value]], + dict: lambda value: ["map", [[canonical(key), canonical(item)] for key, item in value.items()]], +} + + +def canonical(value: object) -> object: + """Return the tagged canonical form the native oracle hooks also produce.""" + + return CANONICAL[type(value)](value) + + +def python_toml_outcome(*, text: str) -> object: + """Return `tomllib`'s canonical value, or the rejection marker when it raises.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress(tomllib.TOMLDecodeError, ValueError): + outcome[0] = canonical(tomllib.loads(text)) + return outcome[0] + + +def native_outcome(payload: str) -> object: + """Return the native canonical value, or the marker of its failure kind.""" + + parsed: object = json.loads(payload) + return {list: parsed, dict: DEFERRED}[type(parsed)] + + +def native_toml_outcome(*, text: str) -> object: + """Return the native TOML loader's outcome for `text`.""" + + return native_outcome(_native._oracle_toml_load(text)) + + +def repository_files(*, pattern: str) -> list[Path]: + """Return tracked-looking repository files matching `pattern`, outside build directories.""" + + return sorted( + filter( + lambda path: EXCLUDED_DIRECTORIES.isdisjoint(path.relative_to(REPOSITORY_ROOT).parts), + REPOSITORY_ROOT.glob(pattern), + ) + ) + + +def _python_project_fields(project: ProjectConfig, local: LocalConfig) -> dict[str, object]: + return { + "name": project.name, + "adapter": project.adapter, + "default_target": project.default_target, + "sql_analysis": project.settings.sql_analysis, + "require_sql_analysis": project.settings.require_sql_analysis, + "enforce_placement": project.scopes.enforce_placement, + "enforce_explicit_references": project.references.enforce_explicit, + "vars": [list(item) for item in project.vars.items()], + "path_default_keys": list(project.path_defaults), + "target_names": list(project.targets), + "local_target": local.target, + "local_adapter": local.adapter, + "local_sql_analysis": {True: local.settings.sql_analysis, False: None}[ + "sql_analysis" in local.setting_overrides + ], + "local_vars": [list(item) for item in local.vars.items()], + "local_target_names": list(local.targets), + } + + +def python_project_outcome(*, project_dir: Path) -> object: + """Return the discovery fields Python's loaders read, or the rejection marker.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress(Exception): + outcome[0] = _python_project_fields( + load_project_config(project_dir=project_dir), load_local_config(project_dir=project_dir) + ) + return outcome[0] + + +def native_project_outcome(*, project_dir: Path) -> object: + """Return the discovery fields the native reader reads, or its failure marker.""" + + parsed: dict[str, object] = json.loads(_native._oracle_project_config(str(project_dir))) + return {True: DEFERRED, False: parsed}["error" in parsed] + + +def deferred_valid_count(*, expected: list[object], actual: list[object]) -> int: + """Return how many documents Python accepts but the native reader defers back to Python.""" + + return sum( + map( + lambda python, native: native == DEFERRED and python not in UNCHECKED_PYTHON_OUTCOMES, + expected, + actual, + ) + ) + + +def _is_harmful(expected: object, actual: object) -> bool: + return actual != DEFERRED and actual != expected + + +def harmful_mismatches( + *, inputs: list[object], expected: list[object], actual: list[object] +) -> list[tuple[object, object, object]]: + """Return the first few cases where the native reader accepts a different result. + + A native failure defers the file to the Python loader, so only an accepted value that + differs from Python's outcome, including accepting what Python rejects, is harmful. + """ + + return list( + compress(zip(inputs, expected, actual, strict=True), map(_is_harmful, expected, actual)) + )[:3] diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py new file mode 100644 index 0000000000..243dd3840b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py @@ -0,0 +1,48 @@ +"""Compare the native project configuration reader with SQLBuild's Python loaders.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + deferred_valid_count, + harmful_mismatches, + native_project_outcome, + python_project_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="fixture and example projects", + pattern="**/sqlbuild_project.toml", + expected_minimum_files=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_projects_when_reading_natively_then_discovery_fields_match_python( + test_case: RepositoryFileOracleTestCase, +) -> None: + project_dirs: list[Path] = [path.parent for path in repository_files(pattern=test_case.pattern)] + + expected: list[object] = [python_project_outcome(project_dir=path) for path in project_dirs] + actual: list[object] = [native_project_outcome(project_dir=path) for path in project_dirs] + + assert len(project_dirs) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, project_dirs)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py new file mode 100644 index 0000000000..f44563fea8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py @@ -0,0 +1,128 @@ +"""Compare the native TOML loader with `tomllib` on seeded and repository documents.""" + +from __future__ import annotations + +import random +import time +from collections.abc import Callable +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + GeneratedDocumentOracleTestCase, + HostileDocumentOracleTestCase, + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + DEFERRED, + HOSTILE_TOML_DOCUMENTS, + TOML_DOCUMENT_GENERATORS, + deferred_valid_count, + harmful_mismatches, + native_toml_outcome, + python_toml_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedDocumentOracleTestCase( + description="tables, dotted keys, arrays of tables, strings, numbers and datetimes", + generator="toml", + seed=51, + case_count=5000, + expected_maximum_deferred=350, + ), + GeneratedDocumentOracleTestCase( + description="tables, arrays of tables, inline tables and dotted keys reopening namespaces", + generator="toml namespaces", + seed=52, + case_count=5000, + expected_maximum_deferred=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_toml_when_loading_natively_then_values_match_tomllib( + test_case: GeneratedDocumentOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + generate: Callable[[random.Random], str] = TOML_DOCUMENT_GENERATORS[test_case.generator] + texts: list[str] = [generate(rng) for _ in range(test_case.case_count)] + + expected: list[object] = [python_toml_outcome(text=text) for text in texts] + actual: list[object] = [native_toml_outcome(text=text) for text in texts] + + assert ( + deferred_valid_count(expected=expected, actual=actual) + <= test_case.expected_maximum_deferred + ) + assert harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="repository TOML files", pattern="**/*.toml", expected_minimum_files=8 + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_toml_files_when_loading_natively_then_values_match_tomllib( + test_case: RepositoryFileOracleTestCase, +) -> None: + paths: list[Path] = repository_files(pattern=test_case.pattern) + texts: list[str] = [path.read_text(encoding="utf-8") for path in paths] + + expected: list[object] = [python_toml_outcome(text=text) for text in texts] + actual: list[object] = [native_toml_outcome(text=text) for text in texts] + + assert len(paths) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, paths)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +@pytest.mark.parametrize( + "test_case", + [ + HostileDocumentOracleTestCase( + description="ten thousand nested arrays", document="deep arrays" + ), + HostileDocumentOracleTestCase( + description="ten thousand nested inline tables", document="deep inline tables" + ), + HostileDocumentOracleTestCase( + description="a dotted key with a hundred thousand segments", document="long dotted key" + ), + HostileDocumentOracleTestCase( + description="a table header with a hundred thousand segments", + document="long table header", + ), + ], + ids=lambda case: case.description, +) +def test_given_hostile_toml_when_loading_natively_then_reader_defers_quickly( + test_case: HostileDocumentOracleTestCase, +) -> None: + text: str = HOSTILE_TOML_DOCUMENTS[test_case.document]() + + started: float = time.perf_counter() + outcome: object = native_toml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (outcome == DEFERRED, elapsed < test_case.expected_maximum_seconds) == ( + test_case.expected_deferred, + True, + ), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 6c9cc3e7e9d50209b20d09c2f5eb0dfe9a422a91 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:12:25 +0100 Subject: [PATCH 4/4] feat(native): add a PyYAML-compatible YAML 1.1 loader with oracle tests Composes saphyr-parser events into PyYAML's node graph and emulates the YAML 1.1 resolver and SafeConstructor, including merge keys, base-60 numbers, timestamps and duplicate-key handling. Constructs PyYAML or LibYAML handle differently defer to Python. Oracle tests compare against SafeLoader and CSafeLoader on seeded and repository documents. Literal and folded scalars are read by a port of PyYAML's block scalar scanner, cross-checked against the parser, so chomping matches PyYAML with or without a final line break. A leading byte order mark is skipped. Constructs the two PyYAML loaders read differently, such as tabs, '?' inside flow scalars and directives, are deferred, and the oracle tests require deferral wherever SafeLoader and CSafeLoader disagree. Shared nodes are built once, and construction depth plus a size budget proportional to the input bound alias and merge expansion. Property lookups use code positions computed once per document and mapping keys are hashed by Python equality, so large single-line flow collections load in linear time. --- Cargo.lock | 41 +- Cargo.toml | 1 + crates/README.md | 2 +- crates/sqlbuild-config/Cargo.toml | 3 +- crates/sqlbuild-config/src/lib.rs | 3 +- .../src/yaml/_helpers/anchors.rs | 87 ++++ .../src/yaml/_helpers/block_scalars.rs | 217 ++++++++ .../src/yaml/_helpers/composer.rs | 475 ++++++++++++++++++ .../src/yaml/_helpers/constructor.rs | 274 ++++++++++ .../sqlbuild-config/src/yaml/_helpers/keys.rs | 63 +++ .../sqlbuild-config/src/yaml/_helpers/mod.rs | 10 + .../src/yaml/_helpers/numbers.rs | 196 ++++++++ .../src/yaml/_helpers/patterns.rs | 112 +++++ .../src/yaml/_helpers/reader.rs | 73 +++ .../src/yaml/_helpers/resolver.rs | 113 +++++ .../src/yaml/_helpers/timestamps.rs | 187 +++++++ crates/sqlbuild-config/src/yaml/constants.rs | 62 +++ crates/sqlbuild-config/src/yaml/main/mod.rs | 1 + .../src/yaml/main/safe_load.rs | 12 + crates/sqlbuild-config/src/yaml/mod.rs | 7 + crates/sqlbuild-config/src/yaml/models.rs | 23 + crates/sqlbuild-config/src/yaml/tests.rs | 6 + .../sqlbuild-config/src/yaml/tests/helpers.rs | 73 +++ .../src/yaml/tests/test_safe_load.rs | 438 ++++++++++++++++ .../src/yaml/tests/test_types.rs | 14 + .../src/bindings/_helpers/oracles.rs | 8 + src/sqlbuild/_native.pyi | 1 + .../compiler/discovery/_test_types.py | 9 + .../sqlbuild/compiler/discovery/helpers.py | 239 ++++++++- .../discovery/test_native_yaml_oracle.py | 177 +++++++ 30 files changed, 2920 insertions(+), 7 deletions(-) create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/anchors.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/composer.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/constructor.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/keys.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/numbers.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/patterns.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/reader.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/resolver.rs create mode 100644 crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs create mode 100644 crates/sqlbuild-config/src/yaml/constants.rs create mode 100644 crates/sqlbuild-config/src/yaml/main/mod.rs create mode 100644 crates/sqlbuild-config/src/yaml/main/safe_load.rs create mode 100644 crates/sqlbuild-config/src/yaml/mod.rs create mode 100644 crates/sqlbuild-config/src/yaml/models.rs create mode 100644 crates/sqlbuild-config/src/yaml/tests.rs create mode 100644 crates/sqlbuild-config/src/yaml/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs create mode 100644 crates/sqlbuild-config/src/yaml/tests/test_types.rs create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py diff --git a/Cargo.lock b/Cargo.lock index cc5d0beaaa..8bbfed4ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ dependencies = [ "object", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "autocfg" version = "1.5.1" @@ -337,7 +343,7 @@ dependencies = [ "serde", "serde_json", "stacker", - "thiserror", + "thiserror 1.0.69", "unicode-segmentation", ] @@ -545,6 +551,16 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "saphyr-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7429158804c36e705d9423b7848018e535b3e92d439a763e364ee12670ef473b" +dependencies = [ + "arraydeque", + "thiserror 2.0.21", +] + [[package]] name = "serde" version = "1.0.229" @@ -634,6 +650,7 @@ dependencies = [ name = "sqlbuild-config" version = "0.138.0" dependencies = [ + "saphyr-parser", "tempfile", "toml_edit", "toml_parser", @@ -771,7 +788,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", ] [[package]] @@ -785,6 +811,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2bc33451a9..0b0de3dfde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot. ] } quote = "1" rayon = "1" +saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. diff --git a/crates/README.md b/crates/README.md index 37d21326b7..e184439cbd 100644 --- a/crates/README.md +++ b/crates/README.md @@ -8,7 +8,7 @@ that can be tested and benchmarked with `cargo test` alone. |---|---| | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | -| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | +| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | diff --git a/crates/sqlbuild-config/Cargo.toml b/crates/sqlbuild-config/Cargo.toml index 73c43456a5..3b6b97cb00 100644 --- a/crates/sqlbuild-config/Cargo.toml +++ b/crates/sqlbuild-config/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "sqlbuild-config" version.workspace = true -description = "Project configuration reading for SQLBuild: TOML." +description = "Project configuration reading for SQLBuild: TOML and PyYAML-compatible YAML." edition.workspace = true license.workspace = true publish.workspace = true rust-version.workspace = true [dependencies] +saphyr-parser.workspace = true toml_edit.workspace = true toml_parser.workspace = true diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index 2ba75e1044..bdbf4ddbf3 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -1,4 +1,4 @@ -//! Project configuration reading: `tomllib`-compatible TOML. +//! Project configuration reading: `tomllib`-compatible TOML and PyYAML-compatible YAML. #![forbid(unsafe_code)] @@ -7,3 +7,4 @@ pub mod errors; pub mod models; pub mod project; pub mod toml; +pub mod yaml; diff --git a/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs b/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs new file mode 100644 index 0000000000..856b203850 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs @@ -0,0 +1,87 @@ +//! Read node properties and indicators from the source, which parser events do not carry. + +/// No code character precedes the position. +const NO_CODE: usize = usize::MAX; +const TOKEN_BOUNDARIES: [char; 5] = ['[', ']', '{', '}', ',']; +/// The characters PyYAML's `scan_anchor` accepts right after an anchor name. +const ANCHOR_TERMINATORS: [char; 8] = ['?', ':', ',', ']', '}', '%', '@', '`']; + +/// For each char index, the end of the last code character before it, outside comments. +pub(crate) fn code_ends(chars: &[char]) -> Vec { + let mut ends: Vec = Vec::with_capacity(chars.len() + 1); + let mut last = NO_CODE; + let mut in_comment = false; + for (index, character) in chars.iter().enumerate() { + ends.push(last); + if *character == '\n' { + in_comment = false; + continue; + } + let line_start = index == 0 || chars[index - 1] == '\n'; + if *character == '#' && (line_start || chars[index - 1].is_whitespace()) { + in_comment = true; + } + if !in_comment && !character.is_whitespace() { + last = index + 1; + } + } + ends.push(last); + ends +} + +/// The end of the last code character before `end`, skipping whitespace and comments. +fn code_end_before(code_ends: &[usize], end: usize) -> Option { + let last = code_ends[end.min(code_ends.len() - 1)]; + (last != NO_CODE).then_some(last) +} + +/// The token that ends right before `end`, skipping whitespace and comments, with its start. +fn token_before(chars: &[char], code_ends: &[usize], end: usize) -> Option<(usize, String)> { + let token_end = code_end_before(code_ends, end)?; + let token_start = chars[..token_end] + .iter() + .rposition(|character| character.is_whitespace() || TOKEN_BOUNDARIES.contains(character)) + .map_or(0, |boundary| boundary + 1); + Some((token_start, chars[token_start..token_end].iter().collect())) +} + +/// Whether a standalone `?` explicit key indicator is the last code before char index `end`. +pub(crate) fn explicit_key_before(chars: &[char], code_ends: &[usize], end: usize) -> bool { + code_end_before(code_ends, end).is_some_and(|index| { + let indicator = index - 1; + let standalone = indicator == 0 + || matches!( + chars[indicator - 1], + ' ' | '\t' | '\n' | '\r' | '[' | '{' | ',' + ); + chars[indicator] == '?' && standalone + }) +} + +/// The anchor name in `token` when PyYAML's scanner accepts the character after it. +fn anchor_token(chars: &[char], token_start: usize, token: &str) -> Option { + let name = token.strip_prefix('&')?; + let following = chars.get(token_start + token.chars().count()).copied(); + let terminated = following.is_none_or(|character| { + character.is_whitespace() || ANCHOR_TERMINATORS.contains(&character) + }); + terminated.then(|| name.to_owned()) +} + +/// The name of the anchor property written before the node starting at char index `start`. +pub(crate) fn anchor_name(chars: &[char], code_ends: &[usize], start: usize) -> Option { + let (token_start, token) = token_before(chars, code_ends, start)?; + if token.starts_with('&') { + return anchor_token(chars, token_start, &token); + } + let (previous_start, previous) = token_before(chars, code_ends, token_start)?; + anchor_token(chars, previous_start, &previous) +} + +/// Whether PyYAML's scanner accepts `name` as an anchor: ASCII letters, digits, `-` and `_`. +pub(crate) fn is_python_anchor_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs b/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs new file mode 100644 index 0000000000..5a6a2b4602 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs @@ -0,0 +1,217 @@ +//! PyYAML's `scan_block_scalar`, which reads literal and folded scalars and chomps their tails. + +use crate::constants::BYTE_ORDER_MARK; +use crate::yaml::constants::{KEEP_INDICATOR, STRIP_INDICATOR}; + +const END: char = '\0'; +const FOLDED_INDICATOR: char = '>'; +const LINE_FEED: &str = "\n"; +const DIGITS: [char; 10] = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; + +fn is_break(character: char) -> bool { + matches!(character, '\r' | '\n' | '\u{85}' | '\u{2028}' | '\u{2029}') +} + +fn is_blank(character: char) -> bool { + matches!(character, ' ' | '\t') +} + +/// The column of char index `index`, counted as PyYAML's reader counts it. +pub(crate) fn column_of(chars: &[char], index: usize) -> usize { + let line_start = chars[..index.min(chars.len())] + .iter() + .rposition(|character| is_break(*character)) + .map_or(0, |line_break| line_break + 1); + chars[line_start..index.min(chars.len())] + .iter() + .filter(|character| **character != BYTE_ORDER_MARK) + .count() +} + +/// PyYAML's scanner position over the document characters. +struct BlockScanner<'chars> { + chars: &'chars [char], + index: usize, + column: usize, +} + +impl BlockScanner<'_> { + fn peek(&self, offset: usize) -> char { + self.chars.get(self.index + offset).copied().unwrap_or(END) + } + + fn forward(&mut self, count: usize) { + for _ in 0..count { + let character = self.peek(0); + self.index += 1; + if matches!(character, '\n' | '\u{85}' | '\u{2028}' | '\u{2029}') + || (character == '\r' && self.peek(0) != '\n') + { + self.column = 0; + } else if character != BYTE_ORDER_MARK { + self.column += 1; + } + } + } + + fn line_break(&mut self) -> String { + match self.peek(0) { + '\r' if self.peek(1) == '\n' => { + self.forward(2); + "\n".to_owned() + } + '\r' | '\n' | '\u{85}' => { + self.forward(1); + "\n".to_owned() + } + separator @ ('\u{2028}' | '\u{2029}') => { + self.forward(1); + separator.to_string() + } + _ => String::new(), + } + } + + fn at_line_end(&self) -> bool { + self.peek(0) == END || is_break(self.peek(0)) + } + + /// An indentation digit: `Some(None)` when absent and `None` for the invalid `0`. + fn digit(&mut self) -> Option> { + let Some(increment) = DIGITS.iter().position(|digit| *digit == self.peek(0)) else { + return Some(None); + }; + self.forward(1); + (increment > 0).then_some(Some(increment)) + } + + fn chomping(&mut self) -> Option { + let character = self.peek(0); + let chomping = matches!(character, KEEP_INDICATOR | STRIP_INDICATOR); + chomping.then(|| { + self.forward(1); + character == KEEP_INDICATOR + }) + } + + /// `scan_block_scalar_indicators`: the chomping flag and the explicit indentation increment. + fn indicators(&mut self) -> Option<(Option, Option)> { + let indicators = match self.chomping() { + Some(chomping) => (Some(chomping), self.digit()?), + None => { + let increment = self.digit()?; + (self.chomping(), increment) + } + }; + (self.peek(0) == ' ' || self.at_line_end()).then_some(indicators) + } + + /// `scan_block_scalar_ignored_line`: spaces and a comment after the indicators. + fn ignored_line(&mut self) -> Option<()> { + while self.peek(0) == ' ' { + self.forward(1); + } + if self.peek(0) == '#' { + while !self.at_line_end() { + self.forward(1); + } + } + self.at_line_end().then(|| { + self.line_break(); + }) + } + + /// `scan_block_scalar_indentation`: leading empty lines and the deepest indentation among them. + fn indentation(&mut self) -> (String, usize) { + let mut breaks = String::new(); + let mut max_indent = 0; + while self.peek(0) == ' ' || is_break(self.peek(0)) { + if self.peek(0) == ' ' { + self.forward(1); + max_indent = max_indent.max(self.column); + } else { + breaks.push_str(&self.line_break()); + } + } + (breaks, max_indent) + } + + /// `scan_block_scalar_breaks`: empty lines up to the next line indented to `indent`. + fn breaks(&mut self, indent: usize) -> String { + let mut breaks = String::new(); + while self.column < indent && self.peek(0) == ' ' { + self.forward(1); + } + while is_break(self.peek(0)) { + breaks.push_str(&self.line_break()); + while self.column < indent && self.peek(0) == ' ' { + self.forward(1); + } + } + breaks + } + + fn line(&mut self) -> String { + let start = self.index; + let mut length = 0; + while !(self.peek(length) == END || is_break(self.peek(length))) { + length += 1; + } + self.forward(length); + self.chars[start..start + length].iter().collect() + } +} + +/// Read the block scalar whose indicator is at `indicator`, under a block collection at `parent_indent`. +pub(crate) fn scan_block_scalar( + chars: &[char], + indicator: usize, + parent_indent: Option, +) -> Option<(String, usize)> { + let folded = chars.get(indicator) == Some(&FOLDED_INDICATOR); + let mut scanner = BlockScanner { + chars, + index: indicator, + column: column_of(chars, indicator), + }; + scanner.forward(1); + let (chomping, increment) = scanner.indicators()?; + scanner.ignored_line()?; + let min_indent = parent_indent.map_or(1, |indent| indent + 1); + let (mut breaks, indent) = match increment { + None => { + let (breaks, max_indent) = scanner.indentation(); + (breaks, min_indent.max(max_indent)) + } + Some(increment) => { + let indent = min_indent + increment - 1; + (scanner.breaks(indent), indent) + } + }; + let mut value = String::new(); + let mut line_break = String::new(); + while scanner.column == indent && scanner.peek(0) != END { + value.push_str(&breaks); + let leading_non_space = !is_blank(scanner.peek(0)); + value.push_str(&scanner.line()); + line_break = scanner.line_break(); + breaks = scanner.breaks(indent); + if scanner.column != indent || scanner.peek(0) == END { + break; + } + let joins_with_space = + folded && line_break == LINE_FEED && leading_non_space && !is_blank(scanner.peek(0)); + match (joins_with_space, breaks.is_empty()) { + (true, true) => value.push(' '), + (true, false) => {} + (false, _) => value.push_str(&line_break), + } + } + if chomping != Some(false) { + value.push_str(&line_break); + } + if chomping == Some(true) { + value.push_str(&breaks); + } + Some((value, scanner.index)) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs new file mode 100644 index 0000000000..7d04aaf9d9 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -0,0 +1,475 @@ +//! Compose parser events into one node graph, as PyYAML's composer does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::yaml::_helpers::anchors::{ + anchor_name, code_ends, explicit_key_before, is_python_anchor_name, +}; +use crate::yaml::_helpers::block_scalars::{column_of, scan_block_scalar}; +use crate::yaml::_helpers::resolver::resolve_scalar; +use crate::yaml::constants::{ + FLOW_KEY_INDICATOR, MAP_TAG, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG, + PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, +}; +use crate::yaml::models::{ComposedDocument, Node, NodeContent}; +use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag}; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +type Properties<'input> = (usize, Option>); + +/// The block collection a node belongs to, which sets PyYAML's indentation for block scalars. +#[derive(Clone, Copy, Debug)] +enum Parent { + Document, + /// A block mapping whose key starts at this char index. + MappingAt(usize), + SequenceEntry, +} + +/// Where a node sits: its nesting depth, whether a flow collection encloses it, and whether it is a key. +#[derive(Clone, Copy, Debug)] +struct Placement { + depth: usize, + flow: bool, + key: bool, + parent: Parent, +} + +impl Placement { + fn child(self, flow: bool, key: bool, parent: Parent) -> Self { + Self { + depth: self.depth + 1, + flow: self.flow || flow, + key, + parent, + } + } +} + +/// Composition state of one document. +struct Composer<'input> { + parser: Parser<'input, StrInput<'input>>, + chars: Vec, + code_ends: Vec, + document: ComposedDocument, + anchors: HashMap, + anchor_names: HashSet, + open_anchors: HashSet, + previous_end: usize, + current_end: usize, +} + +fn syntax_error(error: &ScanError) -> ConfigError { + ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.info().to_owned(), + line: Some(error.marker().line()), + column: Some(error.marker().col()), + } +} + +fn full_tag(tag: Option<&Cow<'_, Tag>>) -> Option { + tag.map(|tag| format!("{}{}", tag.handle, tag.suffix)) +} + +fn unsupported(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Unsupported, message) +} + +impl<'input> Composer<'input> { + fn next(&mut self) -> Result<(Event<'input>, Span), ConfigError> { + match self.parser.next_event() { + Some(Ok(event)) => { + self.previous_end = self.current_end; + self.current_end = event.1.end.index(); + Ok(event) + } + Some(Err(error)) => Err(syntax_error(&error)), + None => Err(ConfigError::new( + ConfigErrorKind::Syntax, + "unexpected end of stream", + )), + } + } + + fn push(&mut self, tag: String, content: NodeContent) -> usize { + self.document.nodes.push(Node { tag, content }); + self.document.nodes.len() - 1 + } + + /// Record an anchor before its node's children, rejecting what PyYAML's composer rejects. + fn open_anchor(&mut self, anchor_id: usize, span: Span) -> Result<(), ConfigError> { + if anchor_id == 0 { + return Ok(()); + } + let name = anchor_name(&self.chars, &self.code_ends, span.start.index()) + .ok_or_else(|| unsupported("anchor name could not be recovered"))?; + if !is_python_anchor_name(&name) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + format!("anchor {name:?} has characters PyYAML does not accept"), + )); + } + if !self.anchor_names.insert(name.clone()) { + return Err(ConfigError::new( + ConfigErrorKind::Construct, + format!("found duplicate anchor {name:?}"), + )); + } + self.open_anchors.insert(anchor_id); + Ok(()) + } + + fn close_anchor(&mut self, anchor_id: usize, node: usize) -> usize { + if anchor_id != 0 { + self.open_anchors.remove(&anchor_id); + self.anchors.insert(anchor_id, node); + } + node + } + + fn alias(&self, anchor_id: usize) -> Result { + if self.open_anchors.contains(&anchor_id) { + return Err(unsupported("recursive aliases are left to Python")); + } + self.anchors + .get(&anchor_id) + .copied() + .ok_or_else(|| ConfigError::new(ConfigErrorKind::Syntax, "found undefined alias")) + } + + fn char_at(&self, index: usize) -> Option { + self.chars.get(index).copied() + } + + /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. + fn spans_lines(&self, span: Span) -> bool { + let key_end = span.end.index().min(self.chars.len()); + let indicator = (key_end..self.chars.len()) + .find(|index| !matches!(self.chars[*index], ' ' | '\t' | '\n' | '\r')) + .filter(|index| self.chars[*index] == ':'); + let end = indicator.unwrap_or(key_end); + let mut start = self.previous_end; + while start < end { + match self.chars[start] { + ' ' | '\t' | '\n' | '\r' | ',' | '[' | '{' => start += 1, + '#' => { + while start < end && !matches!(self.chars[start], '\n' | '\r') { + start += 1; + } + } + _ => break, + } + } + let explicit = self.chars.get(start) == Some(&'?') + && matches!( + self.char_at(start + 1), + None | Some(' ' | '\t' | '\n' | '\r') + ); + !explicit + && self.chars[start.min(end)..end] + .iter() + .any(|character| matches!(character, '\n' | '\r')) + } + + /// Whether the collection starting at `span` is written in flow style. + fn is_flow(&self, span: Span) -> bool { + span.end.index() > span.start.index() + && matches!(self.char_at(span.start.index()), Some('[' | '{')) + } + + /// Reject plain scalars PyYAML's scanner reads differently from YAML 1.2. + fn check_plain( + &self, + value: &str, + span: Span, + placement: Placement, + ) -> Result<(), ConfigError> { + let start = span.start.index(); + let first = self.char_at(start); + if placement.flow && first == Some(':') && !value.is_empty() { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar in a flow collection cannot start with ':'", + )); + } + let colon = (span.end.index()..self.chars.len()) + .find(|index| !matches!(self.chars[*index], ' ' | '\t')) + .unwrap_or(self.chars.len()); + let adjacent_value = self.char_at(colon) == Some(':') + && matches!(self.char_at(colon + 1), Some(',' | ']' | '}')); + if placement.flow && adjacent_value { + return Err(unsupported( + "LibYAML rejects ':' directly before a flow indicator", + )); + } + let explicit_key = explicit_key_before(&self.chars, &self.code_ends, start); + if placement.key && value.is_empty() && first == Some(':') && !explicit_key { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a mapping key cannot be empty without '?'", + )); + } + Ok(()) + } + + /// The block scalar indicator between the previous event and the scalar's content. + fn block_indicator(&self, span: Span) -> Option { + let mut index = self.previous_end; + let mut token_start = true; + while index <= span.start.index() { + let character = self.char_at(index)?; + match character { + '|' | '>' => return Some(index), + '#' if token_start => { + while !matches!(self.char_at(index), None | Some('\n' | '\r')) { + index += 1; + } + continue; + } + '!' | '&' if token_start => { + while !matches!(self.char_at(index), None | Some(' ' | '\n' | '\r')) { + index += 1; + } + continue; + } + _ => {} + } + token_start = matches!(character, ' ' | '\n' | '\r'); + index += 1; + } + None + } + + /// The column of the `-` that starts the sequence entry holding the indicator at `indicator`. + fn entry_column(&self, indicator: usize) -> Option { + let mut end = indicator; + loop { + while end > 0 && self.chars[end - 1] == ' ' { + end -= 1; + } + let start = self.chars[..end] + .iter() + .rposition(|character| matches!(character, ' ' | '\n' | '\r')) + .map_or(0, |boundary| boundary + 1); + let token: String = self.chars[start..end].iter().collect(); + if token.starts_with(['!', '&']) { + end = start; + continue; + } + return (token == SEQUENCE_ENTRY_TOKEN && end < indicator) + .then(|| column_of(&self.chars, start)); + } + } + + /// PyYAML's value for a literal or folded scalar, cross-checked against the parser's reading. + fn block_scalar( + &self, + parsed: &str, + span: Span, + placement: Placement, + ) -> Result { + let deferred = || unsupported("this block scalar is left to Python"); + if placement.flow || placement.key { + return Err(deferred()); + } + let indicator = self.block_indicator(span).ok_or_else(deferred)?; + let parent_indent = match placement.parent { + Parent::Document => None, + Parent::MappingAt(key_start) => Some(column_of(&self.chars, key_start)), + Parent::SequenceEntry => Some(self.entry_column(indicator).ok_or_else(deferred)?), + }; + let (value, end) = + scan_block_scalar(&self.chars, indicator, parent_indent).ok_or_else(deferred)?; + let parsed_end = span.end.index(); + let gap = &self.chars[end.min(parsed_end)..end.max(parsed_end).min(self.chars.len())]; + let same_end = gap + .iter() + .all(|character| matches!(character, ' ' | '\n' | '\r')); + let same_content = value.trim_end_matches('\n') == parsed.trim_end_matches('\n'); + (same_end && same_content) + .then_some(value) + .ok_or_else(deferred) + } + + fn compose_node( + &mut self, + event: Event<'input>, + span: Span, + placement: Placement, + ) -> Result { + if placement.depth > MAX_NESTING_DEPTH { + return Err(unsupported("nesting is deeper than the native limit")); + } + match event { + Event::Alias(anchor_id) => self.alias(anchor_id), + Event::Scalar(value, style, anchor_id, tag) => { + if style == ScalarStyle::Plain + && placement.flow + && value.contains(FLOW_KEY_INDICATOR) + { + return Err(unsupported( + "PyYAML ends a flow scalar at '?' and LibYAML does not", + )); + } + if style == ScalarStyle::Plain + && value.starts_with(PLAIN_FORBIDDEN_FIRST_CHARACTERS) + { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar cannot start with this indicator", + )); + } + let flow_value_indicator = placement.flow && value.starts_with(':'); + if style == ScalarStyle::Plain && flow_value_indicator { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar in a flow collection cannot start with ':'", + )); + } + if placement.key && placement.flow && self.spans_lines(span) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "an implicit key in a flow collection must fit on one line", + )); + } + if style == ScalarStyle::Plain && tag.is_none() && anchor_id == 0 { + self.check_plain(&value, span, placement)?; + } + let value = match style { + ScalarStyle::Literal | ScalarStyle::Folded => { + Cow::Owned(self.block_scalar(&value, span, placement)?) + } + _ => value, + }; + let non_specific = full_tag(tag.as_ref()).as_deref() == Some(NON_SPECIFIC_TAG); + if non_specific && value.is_empty() && style == ScalarStyle::Plain { + return Err(unsupported( + "LibYAML and PyYAML differ on an empty '!' scalar", + )); + } + self.open_anchor(anchor_id, span)?; + let tag = scalar_tag(&value, style, full_tag(tag.as_ref())); + let node = self.push(tag, NodeContent::Scalar(value.into_owned())); + Ok(self.close_anchor(anchor_id, node)) + } + Event::SequenceStart(anchor_id, tag) => { + self.compose_sequence((anchor_id, tag), span, placement) + } + Event::MappingStart(anchor_id, tag) => { + self.compose_mapping((anchor_id, tag), span, placement) + } + _ => Err(ConfigError::new(ConfigErrorKind::Syntax, "expected a node")), + } + } + + fn compose_sequence( + &mut self, + properties: Properties<'input>, + span: Span, + placement: Placement, + ) -> Result { + let (anchor_id, tag) = properties; + self.open_anchor(anchor_id, span)?; + let item_placement = placement.child(self.is_flow(span), false, Parent::SequenceEntry); + let mut items: Vec = Vec::new(); + loop { + let (event, item_span) = self.next()?; + if matches!(event, Event::SequenceEnd) { + break; + } + items.push(self.compose_node(event, item_span, item_placement)?); + } + let tag = collection_tag(full_tag(tag.as_ref()), SEQ_TAG); + let node = self.push(tag, NodeContent::Sequence(items)); + Ok(self.close_anchor(anchor_id, node)) + } + + fn compose_mapping( + &mut self, + properties: Properties<'input>, + span: Span, + placement: Placement, + ) -> Result { + let (anchor_id, tag) = properties; + self.open_anchor(anchor_id, span)?; + let flow = self.is_flow(span); + let mut entries: Vec<(usize, usize)> = Vec::new(); + loop { + let (event, key_span) = self.next()?; + if matches!(event, Event::MappingEnd) { + break; + } + let key_parent = Parent::MappingAt(key_span.start.index()); + let key = + self.compose_node(event, key_span, placement.child(flow, true, key_parent))?; + let (value_event, value_span) = self.next()?; + let value_placement = placement.child(flow, false, key_parent); + let value = self.compose_node(value_event, value_span, value_placement)?; + entries.push((key, value)); + } + let tag = collection_tag(full_tag(tag.as_ref()), MAP_TAG); + let node = self.push(tag, NodeContent::Mapping(entries)); + Ok(self.close_anchor(anchor_id, node)) + } + + /// Compose the only document of the stream; a second document is an error, as in PyYAML. + fn compose_stream(mut self) -> Result { + self.next()?; + let (event, _) = self.next()?; + if matches!(event, Event::StreamEnd) { + return Ok(self.document); + } + let (root_event, root_span) = self.next()?; + let root_placement = Placement { + depth: 0, + flow: false, + key: false, + parent: Parent::Document, + }; + let root = self.compose_node(root_event, root_span, root_placement)?; + self.document.root = Some(root); + self.next()?; + let (after, _) = self.next()?; + if !matches!(after, Event::StreamEnd) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "expected a single document in the stream", + )); + } + Ok(self.document) + } +} + +/// PyYAML's tag for a scalar: plain scalars and the `!` tag resolve implicitly. +fn scalar_tag(value: &str, style: ScalarStyle, tag: Option) -> String { + match tag { + Some(tag) if tag != NON_SPECIFIC_TAG => tag, + Some(_) => resolve_scalar(value, true).to_owned(), + None => resolve_scalar(value, style == ScalarStyle::Plain).to_owned(), + } +} + +fn collection_tag(tag: Option, default: &str) -> String { + match tag { + Some(tag) if tag != NON_SPECIFIC_TAG => tag, + _ => default.to_owned(), + } +} + +/// Compose the single document of `text` into a node graph. +pub(crate) fn compose(text: &str) -> Result { + let chars: Vec = text.chars().collect(); + Composer { + parser: Parser::new_from_str(text), + code_ends: code_ends(&chars), + chars, + document: ComposedDocument::default(), + anchors: HashMap::new(), + anchor_names: HashSet::new(), + open_anchors: HashSet::new(), + previous_end: 0, + current_end: 0, + } + .compose_stream() +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs new file mode 100644 index 0000000000..10832a2862 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs @@ -0,0 +1,274 @@ +//! Build values from a composed document, as PyYAML's `SafeConstructor` does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::yaml::_helpers::keys::{is_hashable, key_identity}; +use crate::yaml::_helpers::numbers::{construct_float, construct_int}; +use crate::yaml::_helpers::timestamps::construct_timestamp; +use crate::yaml::constants::{ + BASE_EXPANSION_BUDGET, BOOL_TAG, BOOL_WORDS, EXPANSION_BUDGET_PER_CHARACTER, FLOAT_TAG, + INT_TAG, MAP_TAG, MAX_CONSTRUCTION_DEPTH, MERGE_TAG, NULL_TAG, PYTHON_ONLY_TAGS, SEQ_TAG, + STR_TAG, TIMESTAMP_TAG, VALUE_TAG, +}; +use crate::yaml::models::{ComposedDocument, Node, NodeContent}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; + +/// One mapping entry after merge keys are flattened; `string_key` marks a `=` key read as text. +type FlatEntry = (usize, usize, bool); + +fn construct_error(message: String) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +fn construct_bool(text: &str) -> Result { + let lowered = text.to_lowercase(); + BOOL_WORDS + .iter() + .find(|(word, _)| *word == lowered) + .map(|(_, flag)| ConfigValue::Bool(*flag)) + .ok_or_else(|| construct_error(format!("{lowered:?} is not a YAML boolean"))) +} + +/// SafeConstructor's scalar constructors, by resolved tag. +fn construct_scalar(tag: &str, text: &str) -> Result { + match tag { + NULL_TAG => Ok(ConfigValue::Null), + BOOL_TAG => construct_bool(text), + INT_TAG => construct_int(text), + FLOAT_TAG => construct_float(text), + STR_TAG => Ok(ConfigValue::String(text.to_owned())), + TIMESTAMP_TAG => construct_timestamp(text.strip_suffix('\n').unwrap_or(text)), + _ if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( + ConfigErrorKind::Unsupported, + format!("values tagged {tag} are left to Python"), + )), + SEQ_TAG | MAP_TAG => Err(construct_error(format!( + "expected a collection node for {tag}" + ))), + _ => Err(construct_error(format!( + "could not determine a constructor for the tag {tag:?}" + ))), + } +} + +/// A constructed value and the number of values it holds, itself included. +type Sized = (ConfigValue, usize); + +/// Construction state; values of nodes reached through several aliases are built once. +struct Constructor<'document> { + document: &'document ComposedDocument, + shared: Vec, + built: HashMap, + budget: usize, +} + +fn deferred(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Unsupported, message) +} + +/// The nesting levels below a constructed value, which construction already bounds. +fn height(value: &ConfigValue) -> usize { + match value { + ConfigValue::List(items) => 1 + items.iter().map(height).max().unwrap_or_default(), + ConfigValue::Map(entries) => { + 1 + entries + .iter() + .map(|(key, item)| height(key).max(height(item))) + .max() + .unwrap_or_default() + } + _ => 0, + } +} + +/// Whether each node is reachable from more than one place, through aliases. +fn shared_nodes(document: &ComposedDocument) -> Vec { + let mut references = vec![0_usize; document.nodes.len()]; + for node in &document.nodes { + match &node.content { + NodeContent::Scalar(_) => {} + NodeContent::Sequence(items) => { + for item in items { + references[*item] += 1; + } + } + NodeContent::Mapping(entries) => { + for (key, value) in entries { + references[*key] += 1; + references[*value] += 1; + } + } + } + } + references.into_iter().map(|count| count > 1).collect() +} + +impl Constructor<'_> { + fn node(&self, id: usize) -> &Node { + &self.document.nodes[id] + } + + /// Spend part of the budget, which bounds the values built, copied and merged per document. + fn spend(&mut self, amount: usize) -> Result<(), ConfigError> { + self.budget = self + .budget + .checked_sub(amount) + .ok_or_else(|| deferred("aliases expand beyond the native size budget"))?; + Ok(()) + } + + fn construct(&mut self, id: usize, depth: usize) -> Result { + if depth > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("values nest deeper than the native limit")); + } + if let Some((value, size, height)) = self.built.get(&id) { + if depth + height > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("aliased values nest deeper than the native limit")); + } + let (value, size) = (value.clone(), *size); + self.spend(size)?; + return Ok((value, size)); + } + self.spend(1)?; + let built = self.construct_new(id, depth)?; + if self.shared[id] { + let (value, size) = &built; + self.built.insert(id, (value.clone(), *size, height(value))); + } + Ok(built) + } + + fn construct_new(&mut self, id: usize, depth: usize) -> Result { + let node = self.node(id); + match (&node.content, node.tag.as_str()) { + (NodeContent::Scalar(text), tag) => construct_scalar(tag, text).map(|value| (value, 1)), + (NodeContent::Sequence(items), SEQ_TAG) => { + let items = items.clone(); + let mut values: Vec = Vec::with_capacity(items.len()); + let mut size = 1; + for item in items { + let (value, item_size) = self.construct(item, depth + 1)?; + values.push(value); + size += item_size; + } + Ok((ConfigValue::List(values), size)) + } + (NodeContent::Mapping(_), MAP_TAG) => self.construct_mapping(id, depth), + (_, tag) if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( + ConfigErrorKind::Unsupported, + format!("values tagged {tag} are left to Python"), + )), + (_, tag) => Err(construct_error(format!( + "could not construct a collection tagged {tag:?}" + ))), + } + } + + /// The entries of a mapping node; merge keys accept only mappings and lists of mappings. + fn entries(&self, id: usize) -> Result, ConfigError> { + match &self.node(id).content { + NodeContent::Mapping(entries) => Ok(entries.clone()), + _ => Err(construct_error( + "expected a mapping or list of mappings for merging".to_owned(), + )), + } + } + + /// The mappings a merge key's value contributes, in PyYAML's order. + fn merged(&mut self, value: usize, depth: usize) -> Result, ConfigError> { + let entries = match &self.node(value).content { + NodeContent::Mapping(_) => self.flatten(value, depth + 1)?, + NodeContent::Sequence(items) => { + let mut parts: Vec> = Vec::new(); + for item in items.clone() { + self.entries(item)?; + parts.push(self.flatten(item, depth + 1)?); + } + parts.into_iter().rev().flatten().collect() + } + NodeContent::Scalar(_) => { + return Err(construct_error( + "expected a mapping or list of mappings for merging".to_owned(), + )); + } + }; + self.spend(entries.len())?; + Ok(entries) + } + + /// PyYAML's `flatten_mapping`: merged entries first, then the mapping's own entries. + fn flatten(&mut self, id: usize, depth: usize) -> Result, ConfigError> { + if depth > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("merge keys nest deeper than the native limit")); + } + let mut merged: Vec = Vec::new(); + let mut own: Vec = Vec::new(); + for (key, value) in self.entries(id)? { + match self.node(key).tag.as_str() { + MERGE_TAG => merged.extend(self.merged(value, depth)?), + VALUE_TAG => own.push((key, value, true)), + _ => own.push((key, value, false)), + } + } + merged.extend(own); + Ok(merged) + } + + fn construct_key( + &mut self, + key: usize, + string_key: bool, + depth: usize, + ) -> Result { + match (&self.node(key).content, string_key) { + (NodeContent::Scalar(text), true) => { + let text = text.clone(); + self.spend(1)?; + Ok((ConfigValue::String(text), 1)) + } + _ => self.construct(key, depth + 1), + } + } + + /// A Python dictionary: a repeated key keeps its first position and takes the last value. + fn construct_mapping(&mut self, id: usize, depth: usize) -> Result { + let mut entries: Vec<(ConfigValue, ConfigValue)> = Vec::new(); + let mut size = 1; + let mut positions: HashMap = HashMap::new(); + for (key, value, string_key) in self.flatten(id, depth)? { + let (key, key_size) = self.construct_key(key, string_key, depth)?; + if !is_hashable(&key) { + return Err(construct_error("found unhashable key".to_owned())); + } + let (value, value_size) = self.construct(value, depth + 1)?; + size += key_size + value_size; + match positions.entry(key_identity(&key)) { + Entry::Occupied(position) => entries[*position.get()].1 = value, + Entry::Vacant(position) => { + position.insert(entries.len()); + entries.push((key, value)); + } + } + } + Ok((ConfigValue::Map(entries), size)) + } +} + +/// The value of a composed document; an empty stream is `None` in Python, here `Null`. +pub(crate) fn construct_document( + document: &ComposedDocument, + text_length: usize, +) -> Result { + let Some(root) = document.root else { + return Ok(ConfigValue::Null); + }; + let mut constructor = Constructor { + document, + shared: shared_nodes(document), + built: HashMap::new(), + budget: BASE_EXPANSION_BUDGET + .saturating_add(text_length.saturating_mul(EXPANSION_BUDGET_PER_CHARACTER)), + }; + constructor.construct(root, 0).map(|(value, _)| value) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/keys.rs b/crates/sqlbuild-config/src/yaml/_helpers/keys.rs new file mode 100644 index 0000000000..b332e79444 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/keys.rs @@ -0,0 +1,63 @@ +//! Python dictionary key semantics for constructed YAML mappings. + +use crate::constants::FEBRUARY; +use crate::models::{ConfigDateTime, ConfigValue}; +use crate::yaml::constants::{SECONDS_PER_DAY, SECONDS_PER_MINUTE}; + +/// Whether Python can hash the value, so it can be a dictionary key. +pub(crate) fn is_hashable(value: &ConfigValue) -> bool { + !matches!(value, ConfigValue::List(_) | ConfigValue::Map(_)) +} + +/// The exact integer text of an integral float, or `None` for fractions and non-finite floats. +fn integral_float_text(value: f64) -> Option { + let integral = value.is_finite() && value.trunc() == value; + integral.then(|| format!("{:.0}", value + 0.0)) +} + +/// Numeric keys as exact integer text where Python would compare them as equal integers. +fn integer_text(value: &ConfigValue) -> Option { + match value { + ConfigValue::Bool(flag) => Some(u8::from(*flag).to_string()), + ConfigValue::Integer(number) => Some(number.to_string()), + ConfigValue::BigInteger(text) => Some(text.clone()), + ConfigValue::Float(number) => integral_float_text(*number), + _ => None, + } +} + +fn day_number(value: &ConfigDateTime) -> i64 { + let (year, month, day) = ( + i64::from(value.date.year), + i64::from(value.date.month), + i64::from(value.date.day), + ); + let shifted_year = if month <= FEBRUARY { year - 1 } else { year }; + let era_day = (153 * ((month + 9) % 12) + 2) / 5 + day - 1; + shifted_year * 365 + shifted_year / 4 - shifted_year / 100 + shifted_year / 400 + era_day +} + +fn instant(value: &ConfigDateTime) -> (i64, u32) { + let seconds = day_number(value) * i64::from(SECONDS_PER_DAY) + + i64::from(value.time.hour) * 3600 + + i64::from(value.time.minute) * i64::from(SECONDS_PER_MINUTE) + + i64::from(value.time.second) + - i64::from(value.utc_offset_seconds.unwrap_or_default()); + (seconds, value.time.microsecond) +} + +/// Text equal for two hashable keys exactly when Python's `==` holds, as `1 == 1.0 == True`. +pub(crate) fn key_identity(value: &ConfigValue) -> String { + if let Some(integer) = integer_text(value) { + return format!("int:{integer}"); + } + match value { + ConfigValue::String(text) => format!("str:{text}"), + ConfigValue::Float(number) if number.is_nan() => "float:nan".to_owned(), + ConfigValue::Float(number) => format!("float:{}", number.to_bits()), + ConfigValue::DateTime(moment) if moment.utc_offset_seconds.is_some() => { + format!("aware:{:?}", instant(moment)) + } + other => format!("{other:?}"), + } +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/mod.rs b/crates/sqlbuild-config/src/yaml/_helpers/mod.rs new file mode 100644 index 0000000000..6b6b26ca3e --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod anchors; +pub(crate) mod block_scalars; +pub(crate) mod composer; +pub(crate) mod constructor; +pub(crate) mod keys; +pub(crate) mod numbers; +pub(crate) mod patterns; +pub(crate) mod reader; +pub(crate) mod resolver; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs new file mode 100644 index 0000000000..d756cfdcfa --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs @@ -0,0 +1,196 @@ +//! PyYAML's `construct_yaml_int` and `construct_yaml_float` with Python's integer sizes. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::yaml::constants::{PYTHON_INT_MAX_STR_DIGITS, SEXAGESIMAL_BASE, ZERO_TEXT}; + +const LIMB_BASE: u64 = 1_000_000_000; +const LIMB_DIGITS: usize = 9; + +/// A non-negative arbitrary-precision integer in base 10^9 limbs, least significant first. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct Natural { + limbs: Vec, +} + +impl Natural { + fn multiply_add(self, factor: u32, addend: u32) -> Self { + let mut carry = u64::from(addend); + let mut limbs: Vec = self + .limbs + .into_iter() + .map(|limb| { + let total = u64::from(limb) * u64::from(factor) + carry; + carry = total / LIMB_BASE; + u32::try_from(total % LIMB_BASE).unwrap_or_default() + }) + .collect(); + while carry > 0 { + limbs.push(u32::try_from(carry % LIMB_BASE).unwrap_or_default()); + carry /= LIMB_BASE; + } + Self { limbs } + } + + fn add(self, other: &Self) -> Self { + let length = self.limbs.len().max(other.limbs.len()); + let mut carry = 0_u64; + let mut limbs: Vec = (0..length) + .map(|index| { + let total = u64::from(self.limbs.get(index).copied().unwrap_or_default()) + + u64::from(other.limbs.get(index).copied().unwrap_or_default()) + + carry; + carry = total / LIMB_BASE; + u32::try_from(total % LIMB_BASE).unwrap_or_default() + }) + .collect(); + if carry > 0 { + limbs.push(u32::try_from(carry).unwrap_or_default()); + } + Self { limbs } + } + + fn decimal(&self) -> String { + let mut limbs = self.limbs.iter().rev().skip_while(|limb| **limb == 0); + let Some(first) = limbs.next() else { + return "0".to_owned(); + }; + limbs.fold(first.to_string(), |text, limb| { + format!("{text}{limb:0LIMB_DIGITS$}") + }) + } +} + +fn construct_error(message: String) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +/// Python's `int(text, radix)` for the ASCII inputs PyYAML passes after removing underscores. +fn python_int(text: &str, radix: u32) -> Result<(bool, Natural), ConfigError> { + let trimmed = text.trim_matches(|character: char| character.is_ascii_whitespace()); + let (negative, unsigned) = match trimmed.as_bytes().first() { + Some(b'-') => (true, &trimmed[1..]), + Some(b'+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let prefix = match radix { + 2 => ["0b", "0B"], + 8 => ["0o", "0O"], + 16 => ["0x", "0X"], + _ => ["", ""], + }; + let digits = prefix + .iter() + .filter(|prefix| !prefix.is_empty()) + .find_map(|prefix| unsigned.strip_prefix(prefix)) + .unwrap_or(unsigned); + if digits.is_empty() { + return Err(construct_error(format!( + "invalid literal for int() with base {radix}: {text:?}" + ))); + } + let mut value = Natural::default(); + for character in digits.chars() { + let Some(digit) = character.to_digit(radix) else { + return Err(construct_error(format!( + "invalid literal for int() with base {radix}: {text:?}" + ))); + }; + value = value.multiply_add(radix, digit); + } + Ok((negative, value)) +} + +fn integer_value(negative: bool, value: &Natural) -> ConfigValue { + let decimal = value.decimal(); + let signed = if negative && decimal != ZERO_TEXT { + format!("-{decimal}") + } else { + decimal + }; + match signed.parse::() { + Ok(small) => ConfigValue::Integer(small), + Err(_) => ConfigValue::BigInteger(signed), + } +} + +fn split_sign(text: &str) -> Result<(bool, &str), ConfigError> { + match text.as_bytes().first() { + None => Err(construct_error("string index out of range".to_owned())), + Some(b'-') => Ok((true, &text[1..])), + Some(b'+') => Ok((false, &text[1..])), + Some(_) => Ok((false, text)), + } +} + +/// PyYAML's `construct_yaml_int`, including binary, octal, hexadecimal and base 60. +pub(crate) fn construct_int(text: &str) -> Result { + let cleaned = text.replace('_', ""); + let (negative, value) = split_sign(&cleaned)?; + if value.len() > PYTHON_INT_MAX_STR_DIGITS { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "integers longer than Python's string conversion limit are left to Python", + )); + } + let (inner_negative, magnitude) = if value == ZERO_TEXT { + (false, Natural::default()) + } else if let Some(binary) = value.strip_prefix("0b") { + python_int(binary, 2)? + } else if let Some(hexadecimal) = value.strip_prefix("0x") { + python_int(hexadecimal, 16)? + } else if value.starts_with('0') { + python_int(value, 8)? + } else if value.contains(':') { + value + .split(':') + .try_fold((false, Natural::default()), |(_, total), part| { + let (part_negative, digit) = python_int(part, 10)?; + if part_negative { + return Err(construct_error( + "negative sexagesimal digits are left to Python".to_owned(), + )); + } + Ok((false, total.multiply_add(SEXAGESIMAL_BASE, 0).add(&digit))) + })? + } else { + python_int(value, 10)? + }; + Ok(integer_value(negative != inner_negative, &magnitude)) +} + +/// Python's `float(text)` for the ASCII inputs PyYAML passes. +fn python_float(text: &str) -> Result { + text.trim_matches(|character: char| character.is_ascii_whitespace()) + .parse::() + .map_err(|_| construct_error(format!("could not convert string to float: {text:?}"))) +} + +/// Base-60 digits summed from the least significant one, in PyYAML's floating-point order. +fn sexagesimal_float(text: &str) -> Result { + let digits = text + .split(':') + .map(python_float) + .collect::, _>>()?; + let (total, _) = digits + .iter() + .rev() + .fold((0.0, 1.0), |(total, base), digit| { + (total + digit * base, base * f64::from(SEXAGESIMAL_BASE)) + }); + Ok(total) +} + +/// PyYAML's `construct_yaml_float`, including `.inf`, `.nan` and base 60. +pub(crate) fn construct_float(text: &str) -> Result { + let cleaned = text.replace('_', "").to_lowercase(); + let (negative, value) = split_sign(&cleaned)?; + let sign = if negative { -1.0 } else { 1.0 }; + let magnitude = match value { + ".inf" => f64::INFINITY, + ".nan" => return Ok(ConfigValue::Float(f64::NAN)), + _ if value.contains(':') => sexagesimal_float(value)?, + _ => python_float(value)?, + }; + Ok(ConfigValue::Float(sign * magnitude)) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs b/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs new file mode 100644 index 0000000000..d59ff2e3e5 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs @@ -0,0 +1,112 @@ +//! Anchored, hand-written equivalents of PyYAML's regular expressions. + +/// A cursor over ASCII bytes for hand-written, anchored versions of PyYAML's patterns. +#[derive(Clone, Debug)] +pub(crate) struct PatternCursor<'text> { + bytes: &'text [u8], + position: usize, +} + +impl<'text> PatternCursor<'text> { + pub(crate) fn new(text: &'text str) -> Self { + Self { + bytes: text.as_bytes(), + position: 0, + } + } + + /// Whether one alternative consumes all of `text`, like an anchored regular expression. + pub(crate) fn matches_any( + text: &str, + alternatives: &[fn(&mut PatternCursor<'_>) -> bool], + ) -> bool { + alternatives.iter().any(|alternative| { + let mut cursor = PatternCursor::new(text); + alternative(&mut cursor) && cursor.at_end() + }) + } + + pub(crate) fn peek(&self) -> Option { + self.bytes.get(self.position).copied() + } + + pub(crate) fn position(&self) -> usize { + self.position + } + + pub(crate) fn rewind(&mut self, position: usize) { + self.position = position; + } + + /// Exactly one byte accepted by `accept`. + pub(crate) fn one(&mut self, accept: impl Fn(u8) -> bool) -> bool { + let matched = self.peek().is_some_and(accept); + self.position += usize::from(matched); + matched + } + + /// Zero or more bytes accepted by `accept`; always succeeds. + pub(crate) fn many(&mut self, accept: impl Fn(u8) -> bool) -> bool { + while self.one(&accept) {} + true + } + + /// One or more bytes accepted by `accept`. + pub(crate) fn some(&mut self, accept: impl Fn(u8) -> bool) -> bool { + self.one(&accept) && self.many(accept) + } + + /// Up to `limit` bytes accepted by `accept`, returning how many were taken. + pub(crate) fn up_to(&mut self, limit: usize, accept: impl Fn(u8) -> bool) -> usize { + let start = self.position; + while self.position - start < limit && self.one(&accept) {} + self.position - start + } + + pub(crate) fn literal(&mut self, text: &str) -> bool { + let matched = self.bytes[self.position..].starts_with(text.as_bytes()); + self.position += usize::from(matched) * text.len(); + matched + } + + pub(crate) fn any_literal(&mut self, texts: &[&str]) -> bool { + texts.iter().any(|text| self.literal(text)) + } + + /// An optional `-` or `+`; always succeeds. + pub(crate) fn sign(&mut self) -> bool { + self.one(|byte| byte == b'-' || byte == b'+'); + true + } + + /// The optional exponent `[eE][-+][0-9]+`; always succeeds. + pub(crate) fn exponent(&mut self) -> bool { + let start = self.position; + let complete = self.one(|byte| byte == b'e' || byte == b'E') + && self.one(|byte| byte == b'-' || byte == b'+') + && self.some(|byte| byte.is_ascii_digit()); + if !complete { + self.position = start; + } + true + } + + /// `(?::[0-5]?[0-9])+`, one or more sexagesimal groups. + pub(crate) fn sexagesimal_groups(&mut self) -> bool { + let mut groups = 0; + while self.peek() == Some(b':') { + let width = match &self.bytes[self.position + 1..] { + [b'0'..=b'5', b'0'..=b'9', ..] => 2, + [b'0'..=b'9', ..] => 1, + _ => return false, + }; + self.position += 1 + width; + groups += 1; + } + groups > 0 + } + + pub(crate) fn at_end(&self) -> bool { + self.position == self.bytes.len() + } +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/reader.rs b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs new file mode 100644 index 0000000000..9c568414be --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs @@ -0,0 +1,73 @@ +//! PyYAML's reader checks, applied before parsing. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::yaml::constants::{ + DIRECTIVE_INDICATOR, DOCUMENT_END_MARKER, LOADER_DEPENDENT_CHARACTERS, PYTHON_ONLY_LINE_BREAKS, +}; + +/// Whether PyYAML's reader accepts `character` (`Reader.NON_PRINTABLE` does not match it). +fn is_printable(character: char) -> bool { + matches!( + character, + '\t' | '\n' | '\r' | ' '..='~' | '\u{85}' | '\u{a0}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' + ) || character >= '\u{10000}' +} + +/// Whether a line holds only whitespace or a comment. +fn is_blank_or_comment(line: &str) -> bool { + let content = line.trim_start_matches([' ', '\r']); + content.is_empty() || content.starts_with('#') +} + +/// Whether the first token of the stream is a `...` document end marker, which PyYAML rejects. +fn starts_with_document_end(text: &str) -> bool { + text.split('\n') + .find(|line| !is_blank_or_comment(line)) + .and_then(|line| line.strip_prefix(DOCUMENT_END_MARKER)) + .is_some_and(|rest| rest.is_empty() || rest.starts_with([' ', '\r'])) +} + +/// The text PyYAML's scanner reads: it skips one byte order mark at the very start. +pub(crate) fn without_byte_order_mark(text: &str) -> &str { + text.strip_prefix(BYTE_ORDER_MARK).unwrap_or(text) +} + +/// Reject what PyYAML's reader rejects, and defer characters PyYAML and LibYAML read differently. +pub(crate) fn check_characters(text: &str) -> Result<(), ConfigError> { + if let Some(character) = text.chars().find(|character| !is_printable(*character)) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + format!("unacceptable character {:#x}", u32::from(character)), + )); + } + if text.contains(PYTHON_ONLY_LINE_BREAKS) { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR are YAML 1.1 line breaks", + )); + } + if text.contains(LOADER_DEPENDENT_CHARACTERS) { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "tabs and inner byte order marks are read differently by PyYAML and LibYAML", + )); + } + let directive = text.starts_with(DIRECTIVE_INDICATOR) + || text + .split(['\n', '\r']) + .any(|line| line.starts_with(DIRECTIVE_INDICATOR)); + if directive { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "directives are left to Python", + )); + } + if starts_with_document_end(text) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "expected the node content, but found a document end marker", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs b/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs new file mode 100644 index 0000000000..b136c739af --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs @@ -0,0 +1,113 @@ +//! PyYAML's YAML 1.1 implicit resolvers for plain scalars, without regular expressions. + +use crate::yaml::_helpers::patterns::PatternCursor; + +/// A plain-scalar matcher, the tag it resolves to, and the first characters PyYAML tries it for. +type ScalarResolver = (fn(&str) -> bool, &'static str, &'static str); +use crate::yaml::_helpers::timestamps::matches_implicit_timestamp; +use crate::yaml::constants::{ + BOOL_TAG, BOOL_WORDS, FLOAT_TAG, INT_TAG, MERGE_KEY, MERGE_TAG, NULL_TAG, NULL_WORDS, STR_TAG, + TIMESTAMP_TAG, VALUE_KEY, VALUE_TAG, +}; + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +fn is_digit_or_underscore(byte: u8) -> bool { + byte.is_ascii_digit() || byte == b'_' +} + +fn matches_float(text: &str) -> bool { + let alternatives: [fn(&mut PatternCursor<'_>) -> bool; 5] = [ + |cursor| { + cursor.sign() + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.literal(".") + && cursor.many(is_digit_or_underscore) + && cursor.exponent() + }, + |cursor| { + cursor.literal(".") + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.exponent() + }, + |cursor| { + cursor.sign() + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.sexagesimal_groups() + && cursor.literal(".") + && cursor.many(is_digit_or_underscore) + }, + |cursor| cursor.sign() && cursor.any_literal(&[".inf", ".Inf", ".INF"]), + |cursor| cursor.any_literal(&[".nan", ".NaN", ".NAN"]), + ]; + PatternCursor::matches_any(text, &alternatives) +} + +fn matches_int(text: &str) -> bool { + let alternatives: [fn(&mut PatternCursor<'_>) -> bool; 6] = [ + |cursor| { + cursor.sign() + && cursor.literal("0b") + && cursor.some(|byte| matches!(byte, b'0' | b'1' | b'_')) + }, + |cursor| { + cursor.sign() + && cursor.literal("0") + && cursor.some(|byte| matches!(byte, b'0'..=b'7' | b'_')) + }, + |cursor| cursor.sign() && cursor.literal("0"), + |cursor| { + cursor.sign() + && cursor.one(|byte| matches!(byte, b'1'..=b'9')) + && cursor.many(is_digit_or_underscore) + }, + |cursor| { + cursor.sign() + && cursor.literal("0x") + && cursor.some(|byte| byte.is_ascii_hexdigit() || byte == b'_') + }, + |cursor| { + cursor.sign() + && cursor.one(|byte| matches!(byte, b'1'..=b'9')) + && cursor.many(is_digit_or_underscore) + && cursor.sexagesimal_groups() + }, + ]; + PatternCursor::matches_any(text, &alternatives) +} + +/// The tag PyYAML's `Resolver.resolve` gives a scalar; `implicit` is true for plain scalars. +pub(crate) fn resolve_scalar(value: &str, implicit: bool) -> &'static str { + if !implicit { + return STR_TAG; + } + let candidate = value.strip_suffix('\n').unwrap_or(value); + let resolvers: [ScalarResolver; 7] = [ + ( + |text| BOOL_WORDS.iter().any(|(word, _)| *word == text), + BOOL_TAG, + "yYnNtTfFoO", + ), + (matches_float, FLOAT_TAG, "-+0123456789."), + (matches_int, INT_TAG, "-+0123456789"), + (|text| text == MERGE_KEY, MERGE_TAG, "<"), + (|text| NULL_WORDS.contains(&text), NULL_TAG, "~nN"), + (matches_implicit_timestamp, TIMESTAMP_TAG, "0123456789"), + (|text| text == VALUE_KEY, VALUE_TAG, "="), + ]; + let first = value.chars().next(); + resolvers + .iter() + .filter(|(_, tag, first_characters)| { + first.map_or(*tag == NULL_TAG, |character| { + first_characters.contains(character) + }) + }) + .find(|(matches, _, _)| matches(candidate)) + .map_or(STR_TAG, |(_, tag, _)| tag) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs b/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs new file mode 100644 index 0000000000..845c840049 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs @@ -0,0 +1,187 @@ +//! YAML 1.1 timestamps: PyYAML's implicit resolver pattern and `construct_yaml_timestamp`. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::yaml::_helpers::patterns::PatternCursor; +use crate::yaml::constants::{ + MICROSECOND_DIGITS, NEGATIVE_SIGN, SECONDS_PER_DAY, SECONDS_PER_MINUTE, +}; + +/// The `tz_sign`, `tz_hour` and `tz_minute` groups; `Z` has an empty sign. +type ZoneParts<'text> = (&'text str, &'text str, Option<&'text str>); + +/// The groups of PyYAML's `timestamp_regexp`. +#[derive(Default)] +struct TimestampParts<'text> { + year: &'text str, + month: &'text str, + day: &'text str, + time: Option<(&'text str, &'text str, &'text str)>, + fraction: Option<&'text str>, + zone: Option>, +} + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +fn is_blank(byte: u8) -> bool { + byte == b' ' || byte == b'\t' +} + +/// Matches PyYAML's timestamp pattern over one text. +struct TimestampScanner<'text> { + text: &'text str, + cursor: PatternCursor<'text>, +} + +impl<'text> TimestampScanner<'text> { + /// Between `bounds.0` and `bounds.1` digits. + fn digits(&mut self, bounds: (usize, usize)) -> Option<&'text str> { + let start = self.cursor.position(); + let taken = self.cursor.up_to(bounds.1, is_digit); + (taken >= bounds.0).then(|| &self.text[start..start + taken]) + } + + /// `-` followed by between `bounds.0` and `bounds.1` digits. + fn dashed(&mut self, bounds: (usize, usize)) -> Option<&'text str> { + self.cursor.literal("-").then(|| self.digits(bounds))? + } + + /// `:` followed by two digits. + fn sexagesimal(&mut self) -> Option<&'text str> { + self.cursor.literal(":").then(|| self.digits((2, 2)))? + } + + fn time(&mut self) -> Option<(&'text str, &'text str, &'text str)> { + let separated = self.cursor.one(|byte| byte == b'T' || byte == b't') + || self.cursor.up_to(usize::MAX, is_blank) > 0; + if !separated { + return None; + } + Some(( + self.digits((1, 2))?, + self.sexagesimal()?, + self.sexagesimal()?, + )) + } + + fn zone(&mut self) -> Option> { + let start = self.cursor.position(); + self.cursor.many(is_blank); + if self.cursor.literal("Z") { + return Some(("", "", None)); + } + let sign_start = self.cursor.position(); + let zone = if self.cursor.one(|byte| byte == b'-' || byte == b'+') { + self.digits((1, 2)).map(|hour| { + ( + &self.text[sign_start..=sign_start], + hour, + self.sexagesimal(), + ) + }) + } else { + None + }; + if zone.is_none() { + self.cursor.rewind(start); + } + zone + } + + /// Match the constructor pattern; `strict_date` requires the resolver's two-digit date. + fn parse(mut self, strict_date: bool) -> Option> { + let date_digits = if strict_date { (2, 2) } else { (1, 2) }; + let mut parts = TimestampParts { + year: self.digits((4, 4))?, + month: self.dashed(date_digits)?, + day: self.dashed(date_digits)?, + ..TimestampParts::default() + }; + let time_start = self.cursor.position(); + parts.time = self.time(); + if parts.time.is_none() { + self.cursor.rewind(time_start); + return self.cursor.at_end().then_some(parts); + } + if self.cursor.literal(".") { + let start = self.cursor.position(); + self.cursor.many(is_digit); + parts.fraction = Some(&self.text[start..self.cursor.position()]); + } + parts.zone = self.zone(); + self.cursor.at_end().then_some(parts) + } +} + +fn parse_parts(text: &str, strict_date: bool) -> Option> { + TimestampScanner { + text, + cursor: PatternCursor::new(text), + } + .parse(strict_date) +} + +/// Whether PyYAML's implicit timestamp resolver matches a plain scalar. +pub(crate) fn matches_implicit_timestamp(text: &str) -> bool { + match parse_parts(text, false) { + Some(parts) if parts.time.is_none() => parse_parts(text, true).is_some(), + Some(_) => true, + None => false, + } +} + +fn number(text: &str) -> i64 { + text.bytes() + .fold(0, |total, digit| total * 10 + i64::from(digit - b'0')) +} + +fn invalid(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +fn microseconds(fraction: Option<&str>) -> i64 { + let fraction = fraction.unwrap_or_default(); + let truncated = &fraction[..fraction.len().min(MICROSECOND_DIGITS)]; + number(&format!("{truncated:0>) -> Result, ConfigError> { + let Some((sign, hour, minute)) = zone else { + return Ok(None); + }; + let minutes = number(hour) * 60 + minute.map_or(0, number); + let seconds = + i32::try_from(minutes).map_err(|_| invalid("offset is out of range"))? * SECONDS_PER_MINUTE; + if seconds >= SECONDS_PER_DAY { + return Err(invalid("offset must be strictly between -24 and 24 hours")); + } + Ok(Some(if sign == NEGATIVE_SIGN { + -seconds + } else { + seconds + })) +} + +/// PyYAML's `construct_yaml_timestamp`: a date, or a naive or aware datetime. +pub(crate) fn construct_timestamp(text: &str) -> Result { + let parts = parse_parts(text, false).ok_or_else(|| invalid("not a timestamp"))?; + let date = ConfigDate::checked(number(parts.year), number(parts.month), number(parts.day)) + .ok_or_else(|| invalid("the date does not exist"))?; + let Some((hour, minute, second)) = parts.time else { + return Ok(ConfigValue::Date(date)); + }; + let time = ConfigTime::checked( + number(hour), + number(minute), + number(second), + microseconds(parts.fraction), + ) + .ok_or_else(|| invalid("the time does not exist"))?; + Ok(ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: utc_offset(parts.zone)?, + })) +} diff --git a/crates/sqlbuild-config/src/yaml/constants.rs b/crates/sqlbuild-config/src/yaml/constants.rs new file mode 100644 index 0000000000..2a5b16581e --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/constants.rs @@ -0,0 +1,62 @@ +//! YAML tags SafeLoader resolves and constructs. + +pub(crate) const NULL_TAG: &str = "tag:yaml.org,2002:null"; +pub(crate) const BOOL_TAG: &str = "tag:yaml.org,2002:bool"; +pub(crate) const INT_TAG: &str = "tag:yaml.org,2002:int"; +pub(crate) const FLOAT_TAG: &str = "tag:yaml.org,2002:float"; +pub(crate) const STR_TAG: &str = "tag:yaml.org,2002:str"; +pub(crate) const TIMESTAMP_TAG: &str = "tag:yaml.org,2002:timestamp"; +pub(crate) const MERGE_TAG: &str = "tag:yaml.org,2002:merge"; +pub(crate) const VALUE_TAG: &str = "tag:yaml.org,2002:value"; +pub(crate) const SEQ_TAG: &str = "tag:yaml.org,2002:seq"; +pub(crate) const MAP_TAG: &str = "tag:yaml.org,2002:map"; +pub(crate) const NON_SPECIFIC_TAG: &str = "!"; +pub(crate) const PYTHON_ONLY_TAGS: [&str; 4] = [ + "tag:yaml.org,2002:binary", + "tag:yaml.org,2002:set", + "tag:yaml.org,2002:omap", + "tag:yaml.org,2002:pairs", +]; +pub(crate) const MAX_NESTING_DEPTH: usize = 256; +pub(crate) const PYTHON_ONLY_LINE_BREAKS: [char; 3] = ['\u{85}', '\u{2028}', '\u{2029}']; +pub(crate) const BOOL_WORDS: [(&str, bool); 18] = [ + ("yes", true), + ("Yes", true), + ("YES", true), + ("no", false), + ("No", false), + ("NO", false), + ("true", true), + ("True", true), + ("TRUE", true), + ("false", false), + ("False", false), + ("FALSE", false), + ("on", true), + ("On", true), + ("ON", true), + ("off", false), + ("Off", false), + ("OFF", false), +]; +pub(crate) const NULL_WORDS: [&str; 5] = ["", "~", "null", "Null", "NULL"]; +pub(crate) const SEXAGESIMAL_BASE: u32 = 60; +pub(crate) const MICROSECOND_DIGITS: usize = 6; +pub(crate) const SECONDS_PER_MINUTE: i32 = 60; +pub(crate) const SECONDS_PER_DAY: i32 = 86_400; +pub(crate) const MERGE_KEY: &str = "<<"; +pub(crate) const VALUE_KEY: &str = "="; +pub(crate) const NEGATIVE_SIGN: &str = "-"; +pub(crate) const ZERO_TEXT: &str = "0"; +pub(crate) const DOCUMENT_END_MARKER: &str = "..."; +pub(crate) const LOADER_DEPENDENT_CHARACTERS: [char; 2] = ['\t', crate::constants::BYTE_ORDER_MARK]; +pub(crate) const MAX_CONSTRUCTION_DEPTH: usize = 256; +pub(crate) const BASE_EXPANSION_BUDGET: usize = 100_000; +pub(crate) const EXPANSION_BUDGET_PER_CHARACTER: usize = 16; +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; +pub(crate) const FLOW_KEY_INDICATOR: char = '?'; +pub(crate) const KEEP_INDICATOR: char = '+'; +pub(crate) const STRIP_INDICATOR: char = '-'; +pub(crate) const SEQUENCE_ENTRY_TOKEN: &str = "-"; +pub(crate) const DIRECTIVE_INDICATOR: char = '%'; +pub(crate) const PLAIN_FORBIDDEN_FIRST_CHARACTERS: [char; 5] = ['|', '>', '%', '@', '`']; diff --git a/crates/sqlbuild-config/src/yaml/main/mod.rs b/crates/sqlbuild-config/src/yaml/main/mod.rs new file mode 100644 index 0000000000..ed61b475fd --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/main/mod.rs @@ -0,0 +1 @@ +pub mod safe_load; diff --git a/crates/sqlbuild-config/src/yaml/main/safe_load.rs b/crates/sqlbuild-config/src/yaml/main/safe_load.rs new file mode 100644 index 0000000000..cb226e4f9c --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/main/safe_load.rs @@ -0,0 +1,12 @@ +//! Load one YAML document the way PyYAML's `yaml.safe_load` does. + +use crate::errors::ConfigError; +use crate::models::ConfigValue; + +/// The value `yaml.safe_load(text)` returns, with YAML 1.1 implicit typing and merge keys. +pub fn safe_load(text: &str) -> Result { + let text = crate::yaml::_helpers::reader::without_byte_order_mark(text); + crate::yaml::_helpers::reader::check_characters(text)?; + let document = crate::yaml::_helpers::composer::compose(text)?; + crate::yaml::_helpers::constructor::construct_document(&document, text.chars().count()) +} diff --git a/crates/sqlbuild-config/src/yaml/mod.rs b/crates/sqlbuild-config/src/yaml/mod.rs new file mode 100644 index 0000000000..748414121c --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/yaml/models.rs b/crates/sqlbuild-config/src/yaml/models.rs new file mode 100644 index 0000000000..01ade04ca4 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/models.rs @@ -0,0 +1,23 @@ +//! The composed node graph of one YAML document, before construction. + +/// The content of one node; aliases share node ids. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum NodeContent { + Scalar(String), + Sequence(Vec), + Mapping(Vec<(usize, usize)>), +} + +/// One node with its resolved tag, as PyYAML's composer produces it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Node { + pub(crate) tag: String, + pub(crate) content: NodeContent, +} + +/// A composed document; `root` is `None` for an empty stream. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct ComposedDocument { + pub(crate) nodes: Vec, + pub(crate) root: Option, +} diff --git a/crates/sqlbuild-config/src/yaml/tests.rs b/crates/sqlbuild-config/src/yaml/tests.rs new file mode 100644 index 0000000000..b1224a8c91 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_safe_load.rs"] +mod safe_load; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/yaml/tests/helpers.rs b/crates/sqlbuild-config/src/yaml/tests/helpers.rs new file mode 100644 index 0000000000..13097546b3 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/helpers.rs @@ -0,0 +1,73 @@ +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +pub(super) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} + +/// A one-entry mapping `{"a": value}`. +pub(super) fn under_a(value: ConfigValue) -> ConfigValue { + ConfigValue::Map(vec![(text("a"), value)]) +} + +pub(super) fn mapping(entries: Vec<(ConfigValue, ConfigValue)>) -> ConfigValue { + ConfigValue::Map(entries) +} + +pub(super) fn datetime(parts: [u32; 7], utc_offset_seconds: Option) -> ConfigValue { + ConfigValue::DateTime(ConfigDateTime { + date: date_parts(parts[0], parts[1], parts[2]), + time: ConfigTime { + hour: u8::try_from(parts[3]).unwrap_or_default(), + minute: u8::try_from(parts[4]).unwrap_or_default(), + second: u8::try_from(parts[5]).unwrap_or_default(), + microsecond: parts[6], + }, + utc_offset_seconds, + }) +} + +pub(super) fn date(year: u32, month: u32, day: u32) -> ConfigValue { + ConfigValue::Date(date_parts(year, month, day)) +} + +fn date_parts(year: u32, month: u32, day: u32) -> ConfigDate { + ConfigDate { + year: u16::try_from(year).unwrap_or_default(), + month: u8::try_from(month).unwrap_or_default(), + day: u8::try_from(day).unwrap_or_default(), + } +} + +/// `a0` holds a list, and every later key nests the previous one through an alias. +pub(super) fn alias_chain(links: usize) -> String { + (1..links).fold("a0: &a0 [x]\n".to_owned(), |text, link| { + format!("{text}a{link}: &a{link} [*a{}]\n", link - 1) + }) +} + +/// Each level lists the previous level ten times through aliases. +pub(super) fn billion_laughs(levels: usize) -> String { + (1..levels).fold("a0: &a0 \"lol\"\n".to_owned(), |text, level| { + let alias = format!("*a{}", level - 1); + format!( + "{text}a{level}: &a{level} [{}]\n", + vec![alias; 10].join(", ") + ) + }) +} + +/// Each level merges the previous level twice, doubling the merged entries. +pub(super) fn merge_chain(levels: usize) -> String { + (1..levels).fold("m0: &m0 {k0: 1}\n".to_owned(), |text, level| { + let previous = level - 1; + format!("{text}m{level}: &m{level} {{<<: [*m{previous}, *m{previous}], k{level}: 1}}\n") + }) +} + +/// Each anchor wraps the previous one in `depth` flow sequences. +pub(super) fn deep_anchor_chain(links: usize, depth: usize) -> String { + let (open, close) = ("[".repeat(depth), "]".repeat(depth)); + (1..links).fold(format!("a0: &a0 {open}x{close}\n"), |text, link| { + format!("{text}a{link}: &a{link} {open}*a{}{close}\n", link - 1) + }) +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs b/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs new file mode 100644 index 0000000000..ba3ea3bec4 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs @@ -0,0 +1,438 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; +use crate::yaml::main::safe_load::safe_load; +use crate::yaml::tests::helpers::{ + alias_chain, billion_laughs, date, datetime, deep_anchor_chain, mapping, merge_chain, text, + under_a, +}; +use crate::yaml::tests::test_types::{HostileYamlTestCase, SafeLoadTestCase}; + +#[test] +fn given_yaml_documents_when_loading_then_values_match_pyyaml_safe_load() { + let test_cases = [ + SafeLoadTestCase { + description: "YAML 1.1 booleans", + text: "a: yes", + expected_value: Ok(under_a(ConfigValue::Bool(true))), + }, + SafeLoadTestCase { + description: "mixed-case boolean is text", + text: "a: yEs", + expected_value: Ok(under_a(text("yEs"))), + }, + SafeLoadTestCase { + description: "off is false", + text: "a: Off", + expected_value: Ok(under_a(ConfigValue::Bool(false))), + }, + SafeLoadTestCase { + description: "octal integer", + text: "a: 017", + expected_value: Ok(under_a(ConfigValue::Integer(15))), + }, + SafeLoadTestCase { + description: "0o is not YAML 1.1 octal", + text: "a: 0o17", + expected_value: Ok(under_a(text("0o17"))), + }, + SafeLoadTestCase { + description: "binary integer", + text: "a: 0b101", + expected_value: Ok(under_a(ConfigValue::Integer(5))), + }, + SafeLoadTestCase { + description: "hexadecimal integer", + text: "a: 0x1F", + expected_value: Ok(under_a(ConfigValue::Integer(31))), + }, + SafeLoadTestCase { + description: "underscores in integers", + text: "a: 1_000", + expected_value: Ok(under_a(ConfigValue::Integer(1000))), + }, + SafeLoadTestCase { + description: "sexagesimal integer", + text: "a: 190:20:30", + expected_value: Ok(under_a(ConfigValue::Integer(685_230))), + }, + SafeLoadTestCase { + description: "sexagesimal float", + text: "a: -190:20:30.5", + expected_value: Ok(under_a(ConfigValue::Float(-685_230.5))), + }, + SafeLoadTestCase { + description: "eight is not octal", + text: "a: 08", + expected_value: Ok(under_a(text("08"))), + }, + SafeLoadTestCase { + description: "arbitrary precision integer", + text: "a: 99999999999999999999999", + expected_value: Ok(under_a(ConfigValue::BigInteger( + "99999999999999999999999".to_owned(), + ))), + }, + SafeLoadTestCase { + description: "exponent needs a dot and a sign", + text: "a: 1e5", + expected_value: Ok(under_a(text("1e5"))), + }, + SafeLoadTestCase { + description: "float with signed exponent", + text: "a: 1.0e+5", + expected_value: Ok(under_a(ConfigValue::Float(100_000.0))), + }, + SafeLoadTestCase { + description: "float with trailing dot", + text: "a: 5.", + expected_value: Ok(under_a(ConfigValue::Float(5.0))), + }, + SafeLoadTestCase { + description: "signed leading-dot float is text", + text: "a: +.5", + expected_value: Ok(under_a(text("+.5"))), + }, + SafeLoadTestCase { + description: "negative infinity", + text: "a: -.Inf", + expected_value: Ok(under_a(ConfigValue::Float(f64::NEG_INFINITY))), + }, + SafeLoadTestCase { + description: "tilde is null", + text: "a: ~", + expected_value: Ok(under_a(ConfigValue::Null)), + }, + SafeLoadTestCase { + description: "empty value is null", + text: "a:", + expected_value: Ok(under_a(ConfigValue::Null)), + }, + SafeLoadTestCase { + description: "quoted values stay text", + text: "a: 'yes'", + expected_value: Ok(under_a(text("yes"))), + }, + SafeLoadTestCase { + description: "non-specific tag resolves quoted scalars", + text: "a: ! '123'", + expected_value: Ok(under_a(ConfigValue::Integer(123))), + }, + SafeLoadTestCase { + description: "explicit string tag", + text: "a: !!str 1", + expected_value: Ok(under_a(text("1"))), + }, + SafeLoadTestCase { + description: "explicit integer tag parses hexadecimal text", + text: "a: !!int '0x10'", + expected_value: Ok(under_a(ConfigValue::Integer(16))), + }, + SafeLoadTestCase { + description: "date", + text: "a: 2001-12-14", + expected_value: Ok(under_a(date(2001, 12, 14))), + }, + SafeLoadTestCase { + description: "single-digit month is text", + text: "a: 2001-1-4", + expected_value: Ok(under_a(text("2001-1-4"))), + }, + SafeLoadTestCase { + description: "aware timestamp", + text: "a: 2001-12-14t21:59:43.10-05:00", + expected_value: Ok(under_a(datetime( + [2001, 12, 14, 21, 59, 43, 100_000], + Some(-18_000), + ))), + }, + SafeLoadTestCase { + description: "spaced timestamp with short offset", + text: "a: 2001-12-14 21:59:43.10 -5", + expected_value: Ok(under_a(datetime( + [2001, 12, 14, 21, 59, 43, 100_000], + Some(-18_000), + ))), + }, + SafeLoadTestCase { + description: "naive timestamp truncates to microseconds", + text: "a: 2001-12-14 21:59:43.1234567", + expected_value: Ok(under_a(datetime([2001, 12, 14, 21, 59, 43, 123_456], None))), + }, + SafeLoadTestCase { + description: "merge keys put merged entries first", + text: "base: &b {x: 1, y: 2}\nother:\n y: 3\n <<: *b", + expected_value: Ok(mapping(vec![ + ( + text("base"), + mapping(vec![ + (text("x"), ConfigValue::Integer(1)), + (text("y"), ConfigValue::Integer(2)), + ]), + ), + ( + text("other"), + mapping(vec![ + (text("x"), ConfigValue::Integer(1)), + (text("y"), ConfigValue::Integer(3)), + ]), + ), + ])), + }, + SafeLoadTestCase { + description: "merge list prefers earlier mappings", + text: "a:\n <<: [{k: 1}, {k: 2, j: 3}]", + expected_value: Ok(under_a(mapping(vec![ + (text("k"), ConfigValue::Integer(1)), + (text("j"), ConfigValue::Integer(3)), + ]))), + }, + SafeLoadTestCase { + description: "equal numeric keys collapse to the first key", + text: "{1: a, 1.0: b, true: c}", + expected_value: Ok(mapping(vec![(ConfigValue::Integer(1), text("c"))])), + }, + SafeLoadTestCase { + description: "equals sign key is text", + text: "=: 1", + expected_value: Ok(mapping(vec![(text("="), ConfigValue::Integer(1))])), + }, + SafeLoadTestCase { + description: "empty stream is null", + text: "", + expected_value: Ok(ConfigValue::Null), + }, + SafeLoadTestCase { + description: "duplicate anchors are rejected", + text: "a: &x 1\nb: &x 2", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "equals sign value has no constructor", + text: "a: =", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "merge value outside a key has no constructor", + text: "a: <<", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "impossible date is rejected", + text: "a: 2002-02-30", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "unhashable key is rejected", + text: "{[1, 2]: a}", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "several documents are rejected", + text: "--- 1\n--- 2", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "hexadecimal without digits is rejected", + text: "a: 0x_", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "unknown tags are rejected", + text: "a: !custom 1", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "sets are left to Python", + text: "a: !!set {x}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "control characters are rejected", + text: "a: \u{7}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "YAML 1.1 line separators are left to Python", + text: "a: b\u{2028}c", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "recursive aliases are left to Python", + text: "a: &x [*x]", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "literal scalar at the end of input has no final line break", + text: "a: |\n x", + expected_value: Ok(under_a(text("x"))), + }, + SafeLoadTestCase { + description: "folded scalar at the end of input has no final line break", + text: "a: >\n folded", + expected_value: Ok(under_a(text("folded"))), + }, + SafeLoadTestCase { + description: "sequence entry literal at the end of input", + text: "- |\n x", + expected_value: Ok(ConfigValue::List(vec![text("x")])), + }, + SafeLoadTestCase { + description: "trailing indentation without a line break is not content", + text: "a: |\n x\n ", + expected_value: Ok(under_a(text("x\n"))), + }, + SafeLoadTestCase { + description: "keep chomping counts only complete trailing lines", + text: "a: |+\n x\n\n ", + expected_value: Ok(under_a(text("x\n\n"))), + }, + SafeLoadTestCase { + description: "strip chomping drops trailing lines", + text: "a: |-\n x\n\n", + expected_value: Ok(under_a(text("x"))), + }, + SafeLoadTestCase { + description: "a literal of blank lines clips to empty text", + text: "a: |\n \n", + expected_value: Ok(under_a(text(""))), + }, + SafeLoadTestCase { + description: "more-indented trailing spaces are content", + text: "a: |\n x\n ", + expected_value: Ok(under_a(text("x\n "))), + }, + SafeLoadTestCase { + description: "explicit indentation under a nested sequence", + text: "a:\n - |2-\n x\n y\n", + expected_value: Ok(under_a(ConfigValue::List(vec![text(" x\ny")]))), + }, + SafeLoadTestCase { + description: "a leading byte order mark is skipped", + text: "\u{feff}a: 1", + expected_value: Ok(under_a(ConfigValue::Integer(1))), + }, + SafeLoadTestCase { + description: "the non-specific tag keeps a quoted line break as text", + text: "a: ! \"\\n\"", + expected_value: Ok(under_a(text("\n"))), + }, + SafeLoadTestCase { + description: "sexagesimal digits may have leading zeros", + text: "a: 1:05", + expected_value: Ok(under_a(ConfigValue::Integer(65))), + }, + SafeLoadTestCase { + description: "a lone document end marker is rejected", + text: "...", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a document end marker after comments is rejected", + text: "# c\n...", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a flow key spanning lines is rejected", + text: "{a\nb: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a flow key with its value indicator on the next line is rejected", + text: "{a\n: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "an anchor glued to a flow collection is left to Python", + text: "&a{}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "a plain scalar cannot start with a block indicator", + text: "{>-: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a question mark ending a value is not an explicit key", + text: "a: b?\n: c", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "question marks in flow scalars are left to Python", + text: "[a?b]", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "tabs are left to Python", + text: "a: x\ty", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "inner byte order marks are left to Python", + text: "a: x\u{feff}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "directives are left to Python", + text: "%YAML 1.1\n--- a", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + ]; + for test_case in test_cases { + let actual = safe_load(test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hostile_yaml_when_loading_then_native_reader_defers_without_exhausting_resources() { + let test_cases = [ + HostileYamlTestCase { + description: "a ten thousand link alias chain", + text: alias_chain(10_000), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "billion laughs with twelve levels", + text: billion_laughs(12), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "merge keys doubling over thirty levels", + text: merge_chain(30), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "anchors wrapping deep flow nesting", + text: deep_anchor_chain(50, 200), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "a hundred thousand nested flow sequences", + text: format!("a: {}{}", "[".repeat(100_000), "]".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileYamlTestCase { + description: "a hundred thousand nested block sequences", + text: "- ".repeat(100_000), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "an integer beyond Python's string conversion limit", + text: format!("a: {}", "9".repeat(4_301)), + expected_error: ConfigErrorKind::Unsupported, + }, + ]; + for test_case in test_cases { + let actual = safe_load(&test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, + Err(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_types.rs b/crates/sqlbuild-config/src/yaml/tests/test_types.rs new file mode 100644 index 0000000000..8e36a9ebb0 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; + +pub(super) struct SafeLoadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, +} + +pub(super) struct HostileYamlTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_error: ConfigErrorKind, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs index f16f61ab8c..1c422613db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -9,6 +9,7 @@ use sqlbuild_config::models::{ConfigDate, ConfigTime, ConfigValue}; use sqlbuild_config::project::main::read_local_config::read_local_config; use sqlbuild_config::project::main::read_project_config::read_project_config; use sqlbuild_config::toml::main::load_toml::load_toml; +use sqlbuild_config::yaml::main::safe_load::safe_load; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{ JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, @@ -187,6 +188,12 @@ fn outcome(result: Result) -> String { .to_string() } +/// Load YAML natively and return its canonical form, or `{"error": kind}`. +#[pyfunction] +fn _oracle_yaml_load(text: &str) -> String { + outcome(safe_load(text).map(|value| canonical(&value))) +} + /// Load TOML natively and return its canonical form, or `{"error": kind}`. #[pyfunction] fn _oracle_toml_load(text: &str) -> String { @@ -222,6 +229,7 @@ fn _oracle_project_config(project_dir: &str) -> String { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_yaml_load, module)?)?; module.add_function(wrap_pyfunction!(_oracle_toml_load, module)?)?; module.add_function(wrap_pyfunction!(_oracle_project_config, module)?)?; Ok(()) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index add85bee8c..532bca8ae0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -55,5 +55,6 @@ def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] ) -> tuple[str, list[tuple[int, int, int] | None]]: ... +def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... def _oracle_project_config(project_dir: str) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 8967c0f188..445a81334c 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -34,3 +34,12 @@ class HostileDocumentOracleTestCase: document: str expected_deferred: bool = True expected_maximum_seconds: float = 1.0 + + +@dataclass(frozen=True) +class LargeDocumentOracleTestCase: + """A large single-line document that must load in linear time with PyYAML's value.""" + + description: str + document: str + expected_maximum_seconds: float = 2.0 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 8a6581fb43..86242681ed 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -1,4 +1,4 @@ -"""Seeded TOML generators and canonical values for the native configuration oracles.""" +"""Seeded YAML and TOML generators and canonical values for the native configuration oracles.""" from __future__ import annotations @@ -14,6 +14,8 @@ from pathlib import Path from typing import Any +import yaml + from sqlbuild import _native from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -23,8 +25,58 @@ EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".venv", "node_modules", "target", ".git"}) PYTHON_ERROR: dict[str, str] = {"error": "rejected"} DEFERRED: dict[str, str] = {"error": "deferred to Python"} -UNCHECKED_PYTHON_OUTCOMES: tuple[dict[str, str], ...] = (PYTHON_ERROR,) +LOADERS_DISAGREE: dict[str, str] = {"error": "PyYAML and LibYAML disagree"} +UNCHECKED_PYTHON_OUTCOMES: tuple[dict[str, str], ...] = (PYTHON_ERROR, LOADERS_DISAGREE) YAML_LINE_BREAKS: dict[int, None] = {0x85: None, 0x2028: None, 0x2029: None} +SCALAR_FRAGMENTS: tuple[str, ...] = ( + "0", "1", "7", "8", "9", "00", "12", "59", "60", "_", ".", ":", "e", "E", "e+", "E-", + "+", "-", "x", "b", "o", "0x", "0b", "0o", "a", "f", "F", "inf", "Inf", ".inf", ".nan", + "NaN", "yes", "Yes", "NO", "on", "ON", "off", "true", "True", "FALSE", "null", "Null", + "NULL", "~", "<<", "=", "y", "n", "T", "t", " ", "Z", "2001", "-12", "-2", "-14", "T21", + ":59", ":43", ".10", "-05:00", "+5", " -5", "1_0", "é", "#", "'", '"', +) # fmt: skip +KNOWN_SCALARS: tuple[str, ...] = ( + "yes", "No", "oN", "~", "", "<<", "=", "017", "0o17", "08", "0b101", "0x1F", "0x_", + "1_000", "190:20:30", "1:60", "-190:20:30.5", "1e5", "1.0e5", "1.0e+5", ".5", "5.", "+.5", + "-0.0", ".NaN", "-.inf", "2001-12-14", "2001-1-4", "2002-02-30", "2001-12-14t21:59:43.10-05:00", + "2001-12-14 21:59:43.10 -5", "2001-12-14 21:59:43.1234567Z", "2001-12-14T24:00:00", + "2001-12-14 1:02:03 +23:59", "99999999999999999999999", "-0b_1", "0_", "_1", "1__2.3_", +) # fmt: skip +SCALAR_CONTEXTS: tuple[str, ...] = ( + "key: {plain}", + "- {plain}", + "[{plain}, 1]", + "{plain}: value", + "key: '{single}'", + 'key: "{double}"', + "key: ! {plain}", + "key: ! '{single}'", + "key: !!int {plain}", + "key: !!float {plain}", + "key: !!bool {plain}", + "key: !!str {plain}", + "key: !!null {plain}", + "key: !!timestamp {plain}", + "? {plain}\n: 1", +) +DOCUMENT_TEMPLATES: tuple[str, ...] = ( + "base{n}: &anchor{n}\n shared: {a}\n other: {b}\nitem{n}:\n <<: *anchor{n}\n other: {c}\n", + "list{n}:\n - &item{n} {a}\n - *item{n}\n - [{b}, {c}]\n", + "merged{n}:\n <<: [{{x: {a}}}, {{x: {b}, y: {c}}}]\n z: {a}\n", + "flow{n}: {{k{n}: {a}, j{n}: [{b}, {{m: {c}}}]}}\n", + "literal{n}: |\n {a}\n {b}\n\n {c}\nfolded{n}: >-\n {a}\n {b}\n", + "? complex{n}\n: {a}\n# comment {b}\nafter{n}: {c} # trailing\n", + "{a}: first\n{a}: second\n", + "keep{n}: |+\n {a}\n\n {b}\n\n", + "strip{n}: >-\n {a}\n\n {b}\n \n", + "seq{n}:\n- |\n {a}\n- >+\n {b}\n \n", + "nested{n}:\n inner: |\n {a}\n {b}\n \n", + "indented{n}:\n - |2-\n {a}\n {c}\n", + "empty{n}: |\n \n\nafter{n}: {b}\n", +) +DOCUMENT_TAILS: tuple[str, ...] = ( + "", "\n", "\n\n", " ", " ", "\n ", "\n ", "\n \n", " \n ", "\n# end", "\n...\n", +) # fmt: skip BYTE_ORDER_MARKS: tuple[str, ...] = ("",) * 19 + ("\ufeff",) @@ -34,6 +86,149 @@ def yaml_text(*, rng: random.Random, max_length: int) -> str: return random_text(rng=rng, max_length=max_length).translate(YAML_LINE_BREAKS) +def random_scalar(*, rng: random.Random) -> str: + """Return a near-miss plain scalar built from resolver-sensitive fragments.""" + + pieces: list[str] = [rng.choice(SCALAR_FRAGMENTS) for _ in range(rng.randint(1, 4))] + return rng.choice(("".join(pieces), rng.choice(KNOWN_SCALARS))) + + +def scalar_document(*, rng: random.Random) -> str: + """Return a scalar embedded in a plain, quoted or tagged YAML context.""" + + scalar: str = random_scalar(rng=rng) + return rng.choice(SCALAR_CONTEXTS).format( + plain=scalar, single=scalar.replace("'", "''"), double=scalar.replace('"', "'") + ) + + +def anchored_document(*, rng: random.Random) -> str: + """Return a block document with anchors, aliases, merge keys, literals and comments.""" + + return "".join( + rng.choice(DOCUMENT_TEMPLATES).format( + n=index, a=random_scalar(rng=rng), b=random_scalar(rng=rng), c=random_scalar(rng=rng) + ) + for index in range(rng.randint(1, 4)) + ) + + +def _yaml_key(rng: random.Random) -> object: + return rng.choice( + ( + yaml_text(rng=rng, max_length=6), + rng.randint(-5, 5), + rng.random() < 0.5, + None, + datetime.date(2000 + rng.randint(0, 30), rng.randint(1, 12), rng.randint(1, 28)), + yaml_text(rng=rng, max_length=12), + ) + ) + + +def _yaml_datetime(rng: random.Random) -> object: + offset: datetime.timedelta = datetime.timedelta(minutes=rng.randint(-1439, 1439)) + zone: datetime.tzinfo | None = rng.choice((None, datetime.timezone(offset), datetime.UTC)) + return datetime.datetime( + rng.randint(1, 9999), + rng.randint(1, 12), + rng.randint(1, 28), + rng.randint(0, 23), + rng.randint(0, 59), + rng.randint(0, 59), + rng.choice((0, rng.randint(0, 999_999))), + tzinfo=zone, + ) + + +YAML_SCALARS: tuple[Callable[[random.Random], object], ...] = ( + lambda rng: None, + lambda rng: rng.random() < 0.5, + lambda rng: rng.randint(-(2**70), 2**70), + lambda rng: random_float(rng=rng), + lambda rng: yaml_text(rng=rng, max_length=30), + lambda rng: random_scalar(rng=rng), + lambda rng: datetime.date(rng.randint(1, 9999), rng.randint(1, 12), rng.randint(1, 28)), + _yaml_datetime, +) + + +def _yaml_list(rng: random.Random, depth: int) -> object: + return [random_yaml_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 4))] + + +def _yaml_dict(rng: random.Random, depth: int) -> object: + return { + _yaml_key(rng): random_yaml_value(rng=rng, depth=depth - 1) + for _ in range(rng.randint(0, 4)) + } + + +YAML_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + *(lambda rng, depth, scalar=scalar: scalar(rng) for scalar in YAML_SCALARS), + _yaml_list, + _yaml_dict, + _yaml_list, + _yaml_dict, +) + + +def random_yaml_value(*, rng: random.Random, depth: int) -> object: + """Return a random value of the types PyYAML's safe dumper writes.""" + + available: int = len(YAML_SCALARS) + 4 * min(depth, 1) + return rng.choice(YAML_GENERATORS[:available])(rng, depth) + + +def dumped_document(*, rng: random.Random) -> str: + """Return a document written by PyYAML's safe dumper in a random style.""" + + return yaml.safe_dump( + random_yaml_value(rng=rng, depth=3), + default_flow_style=rng.choice((None, True, False)), + default_style=rng.choice((None, None, '"', "'")), + width=rng.choice((12, 80, 4096)), + allow_unicode=rng.random() < 0.5, + explicit_start=rng.random() < 0.3, + sort_keys=False, + ) + + +def _alias_chain(links: int) -> str: + return "a0: &a0 [x]\n" + "".join( + f"a{link}: &a{link} [*a{link - 1}]\n" for link in range(1, links) + ) + + +def _billion_laughs(levels: int) -> str: + return 'a0: &a0 "lol"\n' + "".join( + f"a{level}: &a{level} [{', '.join([f'*a{level - 1}'] * 10)}]\n" + for level in range(1, levels) + ) + + +def _merge_chain(levels: int) -> str: + return "m0: &m0 {k0: 1}\n" + "".join( + f"m{level}: &m{level} {{<<: [*m{level - 1}, *m{level - 1}], k{level}: 1}}\n" + for level in range(1, levels) + ) + + +HOSTILE_YAML_DOCUMENTS: dict[str, Callable[[], str]] = { + "alias chain": lambda: _alias_chain(10_000), + "billion laughs": lambda: _billion_laughs(12), + "merge chain": lambda: _merge_chain(30), + "deep flow": lambda: "a: " + "[" * 100_000 + "]" * 100_000, + "deep block": lambda: "- " * 100_000, + "long integer": lambda: "a: " + "9" * 4_301, +} +LARGE_YAML_DOCUMENTS: dict[str, Callable[[], str]] = { + "flow mapping": lambda: "{" + ", ".join(f"k{index}: {index}" for index in range(100_000)) + "}", + "flow sequence": lambda: "[" + ",".join("1" for _ in range(200_000)) + "]", + "anchored flow sequence": lambda: ( + "[" + ", ".join(f"&a{index} x{index}" for index in range(50_000)) + "]" + ), +} HOSTILE_TOML_DOCUMENTS: dict[str, Callable[[], str]] = { "deep arrays": lambda: "a = " + "[" * 10_000 + "]" * 10_000, "deep inline tables": lambda: "a = " + "{b = " * 10_000 + "}" * 10_000, @@ -42,6 +237,21 @@ def yaml_text(*, rng: random.Random, max_length: int) -> str: } +def finished_document(*, rng: random.Random, text: str) -> str: + """Return `text` with or without its final line break, trailing blank lines or a BOM.""" + + trimmed: str = text.rstrip("\n") + ending: str = rng.choice((text, trimmed, trimmed + rng.choice(DOCUMENT_TAILS))) + return rng.choice(BYTE_ORDER_MARKS) + ending + + +YAML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "scalars": lambda rng: finished_document(rng=rng, text=scalar_document(rng=rng)), + "anchors": lambda rng: finished_document(rng=rng, text=anchored_document(rng=rng)), + "dumped": lambda rng: finished_document(rng=rng, text=dumped_document(rng=rng)), +} + + def _toml_key(rng: random.Random, index: int) -> str: bare: str = f"k{index}_{rng.choice(('a', 'B', '1', '-', '_'))}" quoted: str = json.dumps(yaml_text(rng=rng, max_length=5) + str(index), ensure_ascii=False) @@ -243,6 +453,25 @@ def canonical(value: object) -> object: return CANONICAL[type(value)](value) +def python_yaml_outcome(*, text: str, loader: type[yaml.SafeLoader]) -> object: + """Return PyYAML's canonical value, or the rejection marker when it raises.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress( + yaml.YAMLError, ValueError, TypeError, OverflowError, AttributeError, KeyError, IndexError + ): + outcome[0] = canonical(yaml.load(text, Loader=loader)) + return outcome[0] + + +def expected_yaml_outcome(*, text: str) -> object: + """Return the value both PyYAML loaders agree on, or the marker that native must defer.""" + + pure: object = python_yaml_outcome(text=text, loader=yaml.SafeLoader) + libyaml: object = python_yaml_outcome(text=text, loader=yaml.CSafeLoader) + return {True: pure, False: LOADERS_DISAGREE}[pure == libyaml] + + def python_toml_outcome(*, text: str) -> object: """Return `tomllib`'s canonical value, or the rejection marker when it raises.""" @@ -259,6 +488,12 @@ def native_outcome(payload: str) -> object: return {list: parsed, dict: DEFERRED}[type(parsed)] +def native_yaml_outcome(*, text: str) -> object: + """Return the native YAML loader's outcome for `text`.""" + + return native_outcome(_native._oracle_yaml_load(text)) + + def native_toml_outcome(*, text: str) -> object: """Return the native TOML loader's outcome for `text`.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py new file mode 100644 index 0000000000..132ce747ef --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py @@ -0,0 +1,177 @@ +"""Compare the native YAML loader with PyYAML `safe_load` on seeded and repository documents.""" + +from __future__ import annotations + +import random +import time +from collections.abc import Callable +from pathlib import Path + +import pytest +import yaml + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + GeneratedDocumentOracleTestCase, + HostileDocumentOracleTestCase, + LargeDocumentOracleTestCase, + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + DEFERRED, + HOSTILE_YAML_DOCUMENTS, + LARGE_YAML_DOCUMENTS, + YAML_DOCUMENT_GENERATORS, + deferred_valid_count, + expected_yaml_outcome, + harmful_mismatches, + native_yaml_outcome, + python_yaml_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedDocumentOracleTestCase( + description="near-miss plain, quoted and tagged scalars", + generator="scalars", + seed=41, + case_count=4000, + expected_maximum_deferred=10, + ), + GeneratedDocumentOracleTestCase( + description="anchors, aliases, merge keys, chomped block scalars and comments", + generator="anchors", + seed=42, + case_count=1500, + expected_maximum_deferred=20, + ), + GeneratedDocumentOracleTestCase( + description="documents written by PyYAML's safe dumper in random styles", + generator="dumped", + seed=43, + case_count=1500, + expected_maximum_deferred=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_yaml_when_loading_natively_then_values_match_both_pyyaml_loaders( + test_case: GeneratedDocumentOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + generate: Callable[[random.Random], str] = YAML_DOCUMENT_GENERATORS[test_case.generator] + texts: list[str] = [generate(rng) for _ in range(test_case.case_count)] + + expected: list[object] = [expected_yaml_outcome(text=text) for text in texts] + actual: list[object] = [native_yaml_outcome(text=text) for text in texts] + + assert ( + deferred_valid_count(expected=expected, actual=actual) + <= test_case.expected_maximum_deferred + ) + assert harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="repository YAML files", pattern="**/*.yml", expected_minimum_files=10 + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_yaml_files_when_loading_natively_then_values_match_both_loaders( + test_case: RepositoryFileOracleTestCase, +) -> None: + paths: list[Path] = repository_files(pattern=test_case.pattern) + texts: list[str] = [path.read_text(encoding="utf-8") for path in paths] + + expected: list[object] = [expected_yaml_outcome(text=text) for text in texts] + actual: list[object] = [native_yaml_outcome(text=text) for text in texts] + + assert len(paths) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, paths)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +@pytest.mark.parametrize( + "test_case", + [ + HostileDocumentOracleTestCase( + description="a ten thousand link alias chain", document="alias chain" + ), + HostileDocumentOracleTestCase( + description="billion laughs with twelve levels", document="billion laughs" + ), + HostileDocumentOracleTestCase( + description="merge keys doubling over thirty levels", document="merge chain" + ), + HostileDocumentOracleTestCase( + description="a hundred thousand nested flow sequences", document="deep flow" + ), + HostileDocumentOracleTestCase( + description="a hundred thousand nested block sequences", document="deep block" + ), + HostileDocumentOracleTestCase( + description="an integer beyond Python's string conversion limit", + document="long integer", + ), + ], + ids=lambda case: case.description, +) +def test_given_hostile_yaml_when_loading_natively_then_reader_defers_quickly( + test_case: HostileDocumentOracleTestCase, +) -> None: + text: str = HOSTILE_YAML_DOCUMENTS[test_case.document]() + + started: float = time.perf_counter() + outcome: object = native_yaml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (outcome == DEFERRED, elapsed < test_case.expected_maximum_seconds) == ( + test_case.expected_deferred, + True, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + LargeDocumentOracleTestCase( + description="a hundred thousand keys in one flow mapping line", document="flow mapping" + ), + LargeDocumentOracleTestCase( + description="two hundred thousand items in one flow sequence line", + document="flow sequence", + ), + LargeDocumentOracleTestCase( + description="fifty thousand anchored items in one flow sequence line", + document="anchored flow sequence", + ), + ], + ids=lambda case: case.description, +) +def test_given_large_single_line_yaml_when_loading_natively_then_time_is_linear_and_values_match( + test_case: LargeDocumentOracleTestCase, +) -> None: + text: str = LARGE_YAML_DOCUMENTS[test_case.document]() + expected: object = python_yaml_outcome(text=text, loader=yaml.CSafeLoader) + + started: float = time.perf_counter() + actual: object = native_yaml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (actual, elapsed < test_case.expected_maximum_seconds) == (expected, True), ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])