diff --git a/Cargo.lock b/Cargo.lock index df5e1a2b6d..8bbfed4ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ dependencies = [ "object", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "autocfg" version = "1.5.1" @@ -337,7 +343,7 @@ dependencies = [ "serde", "serde_json", "stacker", - "thiserror", + "thiserror 1.0.69", "unicode-segmentation", ] @@ -545,6 +551,16 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "saphyr-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7429158804c36e705d9423b7848018e535b3e92d439a763e364ee12670ef473b" +dependencies = [ + "arraydeque", + "thiserror 2.0.21", +] + [[package]] name = "serde" version = "1.0.229" @@ -630,6 +646,16 @@ dependencies = [ "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-config" +version = "0.138.0" +dependencies = [ + "saphyr-parser", + "tempfile", + "toml_edit", + "toml_parser", +] + [[package]] name = "sqlbuild-core" version = "0.138.0" @@ -643,6 +669,7 @@ dependencies = [ "pyo3", "serde_json", "sqlbuild-analysis", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-rules", "sqlbuild-sqltext", @@ -761,7 +788,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", ] [[package]] @@ -775,6 +811,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -784,7 +831,7 @@ dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow 0.7.15", @@ -799,6 +846,27 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" @@ -892,6 +960,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index b49ec1db4e..0b0de3dfde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/sqlbuild-core", "crates/sqlbuild-sqltext", + "crates/sqlbuild-config", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", "crates/sqlbuild-python", @@ -15,6 +16,7 @@ resolver = "2" order = [ "sqlbuild-core", "sqlbuild-sqltext", + "sqlbuild-config", "sqlbuild-analysis", "sqlbuild-rules", "sqlbuild-python", @@ -53,12 +55,14 @@ polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot. ] } quote = "1" rayon = "1" +saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } @@ -67,6 +71,8 @@ sqlparser = { version = "0.58", features = ["visitor"] } syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" toml = "0.9" +toml_edit = { version = "0.25", default-features = false, features = ["parse"] } +toml_parser = "1" walkdir = "2" [workspace.lints.rust] diff --git a/crates/README.md b/crates/README.md index 4bc3c8c67c..e184439cbd 100644 --- a/crates/README.md +++ b/crates/README.md @@ -6,8 +6,9 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| -| `sqlbuild-core` | Constants shared by several crates and the panic boundary that turns an unwinding native failure into a named compiler error. | +| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | +| `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Any error means "defer to Python", which re-parses for exact messages. | | `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | @@ -18,9 +19,13 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-sqltext -> sqlbuild-core + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-core ``` +`sqlbuild-config` does not depend on the crates below it today; its place in the order only +fixes which crates may use it. The JSON emitter lives in `sqlbuild-core` so that any layer can +produce text that must equal Python's `json.dumps` output. + Each crate may depend only on crates below it, and may also skip layers. The order is declared in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and `make check-native-layering` (part of `make check-ci`) fails when: diff --git a/crates/sqlbuild-config/Cargo.toml b/crates/sqlbuild-config/Cargo.toml new file mode 100644 index 0000000000..3b6b97cb00 --- /dev/null +++ b/crates/sqlbuild-config/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sqlbuild-config" +version.workspace = true +description = "Project configuration reading for SQLBuild: TOML and PyYAML-compatible YAML." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +saphyr-parser.workspace = true +toml_edit.workspace = true +toml_parser.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-config/src/constants.rs b/crates/sqlbuild-config/src/constants.rs new file mode 100644 index 0000000000..11688decd0 --- /dev/null +++ b/crates/sqlbuild-config/src/constants.rs @@ -0,0 +1,11 @@ +//! Calendar limits of Python's `datetime` types and characters shared by the readers. + +pub(crate) const MIN_YEAR: i64 = 1; +pub(crate) const MAX_YEAR: i64 = 9999; +pub(crate) const MONTHS_PER_YEAR: i64 = 12; +pub(crate) const FEBRUARY: i64 = 2; +pub(crate) const MAX_HOUR: i64 = 23; +pub(crate) const MAX_MINUTE: i64 = 59; +pub(crate) const MAX_SECOND: i64 = 59; +pub(crate) const MAX_MICROSECOND: i64 = 999_999; +pub(crate) const BYTE_ORDER_MARK: char = '\u{feff}'; diff --git a/crates/sqlbuild-config/src/errors.rs b/crates/sqlbuild-config/src/errors.rs new file mode 100644 index 0000000000..972a4c07c9 --- /dev/null +++ b/crates/sqlbuild-config/src/errors.rs @@ -0,0 +1,37 @@ +//! Structured failures of reading configuration files. + +/// Why a configuration document could not be loaded natively. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfigErrorKind { + /// The text is not a valid document; Python's parser rejects it too. + Syntax, + /// The document parses but a value cannot be built, as PyYAML's constructor or `tomllib` reports. + Construct, + /// The document uses a feature the native reader leaves to Python. + Unsupported, + /// A required file does not exist. + Missing, + /// A field SQLBuild reads has the wrong type or an invalid value. + InvalidField, +} + +/// A configuration failure; the Python reader re-parses the file to report its exact message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigError { + pub kind: ConfigErrorKind, + pub message: String, + pub line: Option, + pub column: Option, +} + +impl ConfigError { + /// An error without a position. + pub fn new(kind: ConfigErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + line: None, + column: None, + } + } +} diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs new file mode 100644 index 0000000000..bdbf4ddbf3 --- /dev/null +++ b/crates/sqlbuild-config/src/lib.rs @@ -0,0 +1,10 @@ +//! Project configuration reading: `tomllib`-compatible TOML and PyYAML-compatible YAML. + +#![forbid(unsafe_code)] + +pub(crate) mod constants; +pub mod errors; +pub mod models; +pub mod project; +pub mod toml; +pub mod yaml; diff --git a/crates/sqlbuild-config/src/models.rs b/crates/sqlbuild-config/src/models.rs new file mode 100644 index 0000000000..8e84b2c66c --- /dev/null +++ b/crates/sqlbuild-config/src/models.rs @@ -0,0 +1,101 @@ +//! Configuration values with the value domain of Python's `tomllib` and PyYAML `safe_load`. + +use crate::constants::{ + FEBRUARY, MAX_HOUR, MAX_MICROSECOND, MAX_MINUTE, MAX_SECOND, MAX_YEAR, MIN_YEAR, + MONTHS_PER_YEAR, +}; + +/// A calendar date, Python's `datetime.date`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigDate { + pub year: u16, + pub month: u8, + pub day: u8, +} + +/// A time of day with microseconds, Python's naive `datetime.time`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigTime { + pub hour: u8, + pub minute: u8, + pub second: u8, + pub microsecond: u32, +} + +/// A date and time, Python's `datetime.datetime`, with its UTC offset in seconds when aware. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ConfigDateTime { + pub date: ConfigDate, + pub time: ConfigTime, + pub utc_offset_seconds: Option, +} + +impl ConfigDate { + /// The date, or `None` where Python's `datetime.date` constructor raises. + pub fn checked(year: i64, month: i64, day: i64) -> Option { + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let month_days = match month { + FEBRUARY => 28 + i64::from(leap), + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + let valid = (MIN_YEAR..=MAX_YEAR).contains(&year) + && (1..=MONTHS_PER_YEAR).contains(&month) + && (1..=month_days).contains(&day); + valid.then_some(Self { + year: year as u16, + month: month as u8, + day: day as u8, + }) + } +} + +impl ConfigTime { + /// The time, or `None` where Python's `datetime.time` constructor raises. + pub fn checked(hour: i64, minute: i64, second: i64, microsecond: i64) -> Option { + let valid = (0..=MAX_HOUR).contains(&hour) + && (0..=MAX_MINUTE).contains(&minute) + && (0..=MAX_SECOND).contains(&second) + && (0..=MAX_MICROSECOND).contains(µsecond); + valid.then_some(Self { + hour: hour as u8, + minute: minute as u8, + second: second as u8, + microsecond: microsecond as u32, + }) + } +} + +/// One loaded configuration value; mappings keep document order and Python key equality. +#[derive(Clone, Debug, PartialEq)] +pub enum ConfigValue { + Null, + Bool(bool), + Integer(i64), + BigInteger(String), + Float(f64), + String(String), + Date(ConfigDate), + DateTime(ConfigDateTime), + Time(ConfigTime), + List(Vec), + Map(Vec<(ConfigValue, ConfigValue)>), +} + +impl ConfigValue { + /// The entries of a mapping, or `None` for any other value. + pub fn as_map(&self) -> Option<&[(ConfigValue, ConfigValue)]> { + match self { + Self::Map(entries) => Some(entries), + _ => None, + } + } + + /// The value of a string key in a mapping, as Python's `mapping.get(key)` finds it. + pub fn get(&self, key: &str) -> Option<&ConfigValue> { + self.as_map()? + .iter() + .find(|(candidate, _)| matches!(candidate, Self::String(text) if text == key)) + .map(|(_, value)| value) + } +} diff --git a/crates/sqlbuild-config/src/project/_helpers/fields.rs b/crates/sqlbuild-config/src/project/_helpers/fields.rs new file mode 100644 index 0000000000..af0621fe3c --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/fields.rs @@ -0,0 +1,92 @@ +//! Typed reads of configuration fields with the Python loader's defaults and type checks. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; + +fn invalid(field: &str, expected: &str) -> ConfigError { + ConfigError::new( + ConfigErrorKind::InvalidField, + format!("{field} must be {expected}"), + ) +} + +/// A section as a mapping; a missing section is empty, as in `_coerce_mapping`. +pub(crate) fn section<'value>( + table: &'value ConfigValue, + key: &str, +) -> Result<&'value [(ConfigValue, ConfigValue)], ConfigError> { + match table.get(key) { + None | Some(ConfigValue::Null) => Ok(&[]), + Some(value) => value.as_map().ok_or_else(|| invalid(key, "a mapping")), + } +} + +/// A non-empty string, stripped, as in `_require_str`. +pub(crate) fn required_text(table: &ConfigValue, key: &str) -> Result { + optional_text(table, key)?.ok_or_else(|| invalid(key, "a non-empty string")) +} + +/// An optional non-empty string, stripped, as in `_optional_str`. +pub(crate) fn optional_text(table: &ConfigValue, key: &str) -> Result, ConfigError> { + match table.get(key) { + None | Some(ConfigValue::Null) => Ok(None), + Some(ConfigValue::String(text)) if !text.trim().is_empty() => { + Ok(Some(text.trim().to_owned())) + } + Some(_) => Err(invalid(key, "a non-empty string")), + } +} + +/// An optional boolean with a default, as in `_optional_bool`. +pub(crate) fn flag( + entries: &[(ConfigValue, ConfigValue)], + key: &str, + default: bool, +) -> Result { + match lookup(entries, key) { + None | Some(ConfigValue::Null) => Ok(default), + Some(ConfigValue::Bool(value)) => Ok(*value), + Some(_) => Err(invalid(key, "a boolean")), + } +} + +/// The value of a string key among mapping entries. +pub(crate) fn lookup<'value>( + entries: &'value [(ConfigValue, ConfigValue)], + key: &str, +) -> Option<&'value ConfigValue> { + entries + .iter() + .find(|(candidate, _)| matches!(candidate, ConfigValue::String(text) if text == key)) + .map(|(_, value)| value) +} + +/// The text of a mapping key; TOML keys are always strings. +pub(crate) fn key_text(key: &ConfigValue) -> Result { + match key { + ConfigValue::String(text) => Ok(text.clone()), + _ => Err(invalid("a key", "a string")), + } +} + +/// String values, as in `_load_string_mapping`. +pub(crate) fn text_mapping( + table: &ConfigValue, + key: &str, +) -> Result, ConfigError> { + section(table, key)? + .iter() + .map(|(name, value)| match value { + ConfigValue::String(text) => Ok((key_text(name)?, text.clone())), + _ => Err(invalid(&format!("{key}.{}", key_text(name)?), "a string")), + }) + .collect() +} + +/// The keys of a section, in document order. +pub(crate) fn section_keys(table: &ConfigValue, key: &str) -> Result, ConfigError> { + section(table, key)? + .iter() + .map(|(name, _)| key_text(name)) + .collect() +} diff --git a/crates/sqlbuild-config/src/project/_helpers/files.rs b/crates/sqlbuild-config/src/project/_helpers/files.rs new file mode 100644 index 0000000000..c34d90dd4e --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/files.rs @@ -0,0 +1,35 @@ +//! Locate and load configuration files as the Python loader does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::toml::main::load_toml::load_toml; +use std::path::Path; + +/// Reject a legacy YAML file that has no TOML replacement, as the Python loader does. +pub(crate) fn reject_legacy( + directory: &Path, + filename: &str, + legacy_filename: &str, +) -> Result<(), ConfigError> { + if !directory.join(filename).exists() && directory.join(legacy_filename).exists() { + return Err(ConfigError::new( + ConfigErrorKind::InvalidField, + format!("{legacy_filename} is no longer supported; rename it to {filename}"), + )); + } + Ok(()) +} + +/// Read and parse one TOML file the way `tomllib.load` does. +pub(crate) fn load_toml_file(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|error| { + ConfigError::new( + ConfigErrorKind::Missing, + format!("{}: {error}", path.display()), + ) + })?; + let text = String::from_utf8(bytes).map_err(|error| { + ConfigError::new(ConfigErrorKind::Syntax, format!("invalid UTF-8: {error}")) + })?; + load_toml(&text) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/mod.rs b/crates/sqlbuild-config/src/project/_helpers/mod.rs new file mode 100644 index 0000000000..b21adf99f8 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod fields; +pub(crate) mod files; diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs new file mode 100644 index 0000000000..850458ae50 --- /dev/null +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -0,0 +1,9 @@ +//! Project configuration file names and keys. + +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation"; +pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs new file mode 100644 index 0000000000..846d5991cb --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -0,0 +1,2 @@ +pub mod read_local_config; +pub mod read_project_config; diff --git a/crates/sqlbuild-config/src/project/main/read_local_config.rs b/crates/sqlbuild-config/src/project/main/read_local_config.rs new file mode 100644 index 0000000000..70d30f2e34 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/read_local_config.rs @@ -0,0 +1,40 @@ +//! Read the discovery fields of `sqlbuild_local.toml`. + +use crate::errors::ConfigError; +use crate::project::_helpers::fields::{ + flag, lookup, optional_text, section, section_keys, text_mapping, +}; +use crate::project::_helpers::files::{load_toml_file, reject_legacy}; +use crate::project::constants::{ + LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, LOCAL_CONFIG_FILENAME, + SQL_ANALYSIS_KEY, +}; +use crate::project::models::LocalConfigFile; +use std::path::Path; + +/// Read `sqlbuild_local.toml` in `project_dir`, or the defaults when it does not exist. +pub fn read_local_config(project_dir: &Path) -> Result { + reject_legacy( + project_dir, + LOCAL_CONFIG_FILENAME, + LEGACY_LOCAL_CONFIG_FILENAME, + )?; + let path = project_dir.join(LOCAL_CONFIG_FILENAME); + if !path.exists() { + return Ok(LocalConfigFile::default()); + } + let table = load_toml_file(&path)?; + let settings = section(&table, "settings")?; + let analysis_key = [SQL_ANALYSIS_KEY, LEGACY_SQL_VALIDATION_KEY] + .into_iter() + .find(|key| lookup(settings, key).is_some()); + Ok(LocalConfigFile { + target: optional_text(&table, "target")?, + adapter: optional_text(&table, "adapter")?, + sql_analysis: analysis_key + .map(|key| flag(settings, key, true)) + .transpose()?, + vars: text_mapping(&table, "vars")?, + target_names: section_keys(&table, "targets")?, + }) +} diff --git a/crates/sqlbuild-config/src/project/main/read_project_config.rs b/crates/sqlbuild-config/src/project/main/read_project_config.rs new file mode 100644 index 0000000000..1c345a5b16 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/read_project_config.rs @@ -0,0 +1,76 @@ +//! Read the discovery fields of `sqlbuild_project.toml`. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::project::_helpers::fields::{ + flag, key_text, lookup, optional_text, required_text, section, section_keys, text_mapping, +}; +use crate::project::_helpers::files::{load_toml_file, reject_legacy}; +use crate::project::constants::{ + LEGACY_PROJECT_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, PROJECT_CONFIG_FILENAME, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, +}; +use crate::project::models::{ProjectConfigFile, ProjectSettings}; +use std::path::Path; + +fn settings(table: &ConfigValue) -> Result { + let entries = section(table, "settings")?; + let current = lookup(entries, SQL_ANALYSIS_KEY); + let legacy = lookup(entries, LEGACY_SQL_VALIDATION_KEY); + if current.is_some() && legacy.is_some() && current != legacy { + return Err(ConfigError::new( + ConfigErrorKind::InvalidField, + "settings.sql_analysis conflicts with legacy settings.sql_validation", + )); + } + let analysis_key = if current.is_some() { + SQL_ANALYSIS_KEY + } else { + LEGACY_SQL_VALIDATION_KEY + }; + Ok(ProjectSettings { + sql_analysis: flag(entries, analysis_key, true)?, + require_sql_analysis: flag(entries, REQUIRE_SQL_ANALYSIS_KEY, false)?, + }) +} + +fn path_defaults(table: &ConfigValue) -> Result, ConfigError> { + section(table, "path_defaults")? + .iter() + .map(|(key, value)| { + let name = key_text(key)?; + match value { + ConfigValue::Map(_) => Ok((name, value.clone())), + _ => Err(ConfigError::new( + ConfigErrorKind::InvalidField, + format!("path_defaults['{name}'] must be a mapping"), + )), + } + }) + .collect() +} + +/// Read `sqlbuild_project.toml` in `project_dir`; Python's loader remains the full validator. +pub fn read_project_config(project_dir: &Path) -> Result { + reject_legacy( + project_dir, + PROJECT_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, + )?; + let table = load_toml_file(&project_dir.join(PROJECT_CONFIG_FILENAME))?; + Ok(ProjectConfigFile { + name: required_text(&table, "name")?, + adapter: required_text(&table, "adapter")?, + default_target: optional_text(&table, "default_target")?, + settings: settings(&table)?, + enforce_placement: flag(section(&table, "scopes")?, "enforce_placement", true)?, + enforce_explicit_references: flag( + section(&table, "references")?, + "enforce_explicit", + true, + )?, + vars: text_mapping(&table, "vars")?, + path_defaults: path_defaults(&table)?, + target_names: section_keys(&table, "targets")?, + }) +} diff --git a/crates/sqlbuild-config/src/project/mod.rs b/crates/sqlbuild-config/src/project/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-config/src/project/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs new file mode 100644 index 0000000000..5cdcb3fff1 --- /dev/null +++ b/crates/sqlbuild-config/src/project/models.rs @@ -0,0 +1,34 @@ +//! The project configuration fields native discovery reads. + +use crate::models::ConfigValue; + +/// Shared settings discovery reads from `[settings]`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectSettings { + pub sql_analysis: bool, + pub require_sql_analysis: bool, +} + +/// The discovery fields of `sqlbuild_project.toml`, with Python's defaults and stripping. +#[derive(Clone, Debug, PartialEq)] +pub struct ProjectConfigFile { + pub name: String, + pub adapter: String, + pub default_target: Option, + pub settings: ProjectSettings, + pub enforce_placement: bool, + pub enforce_explicit_references: bool, + pub vars: Vec<(String, String)>, + pub path_defaults: Vec<(String, ConfigValue)>, + pub target_names: Vec, +} + +/// The discovery fields of `sqlbuild_local.toml`; an absent file reads as the default. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LocalConfigFile { + pub target: Option, + pub adapter: Option, + pub sql_analysis: Option, + pub vars: Vec<(String, String)>, + pub target_names: Vec, +} diff --git a/crates/sqlbuild-config/src/project/tests.rs b/crates/sqlbuild-config/src/project/tests.rs new file mode 100644 index 0000000000..efd9060dc5 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_read_config.rs"] +mod read_config; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/project/tests/helpers.rs b/crates/sqlbuild-config/src/project/tests/helpers.rs new file mode 100644 index 0000000000..b7bb9560e6 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/helpers.rs @@ -0,0 +1,50 @@ +use crate::models::ConfigValue; +use crate::project::models::{ProjectConfigFile, ProjectSettings}; +use tempfile::TempDir; + +/// A project directory containing `files`. +pub(super) fn project_directory(files: &[(&str, &str)]) -> TempDir { + let directory = TempDir::new().expect("temporary directory"); + for (name, contents) in files { + std::fs::write(directory.path().join(name), contents).expect("write project file"); + } + directory +} + +pub(super) fn minimal_project() -> ProjectConfigFile { + ProjectConfigFile { + name: "orders".to_owned(), + adapter: "duckdb".to_owned(), + default_target: None, + settings: ProjectSettings { + sql_analysis: true, + require_sql_analysis: false, + }, + enforce_placement: true, + enforce_explicit_references: true, + vars: vec![], + path_defaults: vec![], + target_names: vec![], + } +} + +pub(super) fn configured_project() -> ProjectConfigFile { + ProjectConfigFile { + default_target: Some("dev".to_owned()), + settings: ProjectSettings { + sql_analysis: false, + require_sql_analysis: true, + }, + enforce_placement: false, + vars: vec![("region".to_owned(), "eu".to_owned())], + path_defaults: vec![( + "staging".to_owned(), + ConfigValue::Map(vec![( + ConfigValue::String("materialized".to_owned()), + ConfigValue::String("view".to_owned()), + )]), + )], + target_names: vec!["dev".to_owned(), "prod".to_owned()], + ..minimal_project() + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_read_config.rs b/crates/sqlbuild-config/src/project/tests/test_read_config.rs new file mode 100644 index 0000000000..5d8534ec76 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_read_config.rs @@ -0,0 +1,125 @@ +use crate::errors::ConfigErrorKind; +use crate::project::main::read_local_config::read_local_config; +use crate::project::main::read_project_config::read_project_config; +use crate::project::models::{LocalConfigFile, ProjectSettings}; +use crate::project::tests::helpers::{configured_project, minimal_project, project_directory}; +use crate::project::tests::test_types::{LocalConfigTestCase, ProjectConfigTestCase}; + +const PROJECT: &str = "sqlbuild_project.toml"; +const LOCAL: &str = "sqlbuild_local.toml"; + +#[test] +fn given_project_files_when_reading_then_discovery_fields_match_python_loader() { + let test_cases = [ + ProjectConfigTestCase { + description: "defaults apply and strings are stripped", + files: &[(PROJECT, "name = ' orders '\nadapter = \"duckdb\"\n")], + expected_config: Ok(minimal_project()), + }, + ProjectConfigTestCase { + description: "discovery sections are read in document order", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\ndefault_target = 'dev'\n\ + [settings]\nsql_analysis = false\nrequire_sql_analysis = true\n\ + [scopes]\nenforce_placement = false\n[vars]\nregion = 'eu'\n\ + [path_defaults.staging]\nmaterialized = 'view'\n\ + [targets.dev]\n[targets.prod]\n", + )], + expected_config: Ok(configured_project()), + }, + ProjectConfigTestCase { + description: "legacy sql_validation sets SQL analysis", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_validation = false\n", + )], + expected_config: Ok(crate::project::models::ProjectConfigFile { + settings: ProjectSettings { + sql_analysis: false, + require_sql_analysis: false, + }, + ..minimal_project() + }), + }, + ProjectConfigTestCase { + description: "conflicting analysis keys are rejected", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_analysis = true\nsql_validation = false\n", + )], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "blank name is rejected", + files: &[(PROJECT, "name = ' '\nadapter = 'duckdb'\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "non-string var is rejected", + files: &[( + PROJECT, + "name = 'orders'\nadapter = 'duckdb'\n[vars]\nlimit = 3\n", + )], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "legacy YAML project file is rejected", + files: &[("sqlbuild_project.yml", "name: orders\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ProjectConfigTestCase { + description: "missing project file", + files: &[], + expected_config: Err(ConfigErrorKind::Missing), + }, + ]; + for test_case in test_cases { + let directory = project_directory(test_case.files); + let actual = read_project_config(directory.path()).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_local_files_when_reading_then_overrides_match_python_loader() { + let test_cases = [ + LocalConfigTestCase { + description: "absent local file reads as defaults", + files: &[], + expected_config: Ok(LocalConfigFile::default()), + }, + LocalConfigTestCase { + description: "local overrides are read", + files: &[( + LOCAL, + "target = 'dev'\n[settings]\nsql_validation = false\n[vars]\nregion = 'us'\n[targets.dev]\n", + )], + expected_config: Ok(LocalConfigFile { + target: Some("dev".to_owned()), + adapter: None, + sql_analysis: Some(false), + vars: vec![("region".to_owned(), "us".to_owned())], + target_names: vec!["dev".to_owned()], + }), + }, + LocalConfigTestCase { + description: "legacy YAML local file is rejected", + files: &[("sqlbuild_local.yml", "target: dev\n")], + expected_config: Err(ConfigErrorKind::InvalidField), + }, + ]; + for test_case in test_cases { + let directory = project_directory(test_case.files); + let actual = read_local_config(directory.path()).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_config, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs new file mode 100644 index 0000000000..f234c337c6 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::project::models::{LocalConfigFile, ProjectConfigFile}; + +pub(super) struct ProjectConfigTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_config: Result, +} + +pub(super) struct LocalConfigTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_config: Result, +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/conversion.rs b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs new file mode 100644 index 0000000000..5486a937ad --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/conversion.rs @@ -0,0 +1,171 @@ +//! Convert a parsed TOML document into the values `tomllib` returns. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use toml_edit::{Array, Datetime, Document, InlineTable, Item, Key, Offset, Table, Value}; + +const NANOSECONDS_PER_MICROSECOND: u32 = 1_000; +/// No key inside: such values sort by their own key alone. +const NO_POSITION: usize = usize::MAX; +/// The deepest value nesting converted natively; the version check keeps documents well within it. +const MAX_DEPTH: usize = 512; +const SECONDS_PER_MINUTE: i32 = 60; + +fn invalid_datetime(message: &str) -> ConfigError { + ConfigError::new( + ConfigErrorKind::Syntax, + format!("invalid date or datetime: {message}"), + ) +} + +fn date(value: toml_edit::Date) -> Result { + ConfigDate::checked( + i64::from(value.year), + i64::from(value.month), + i64::from(value.day), + ) + .ok_or_else(|| invalid_datetime("the date does not exist")) +} + +fn time(value: toml_edit::Time) -> Result { + let second = value + .second + .ok_or_else(|| invalid_datetime("seconds are required before TOML 1.1"))?; + ConfigTime::checked( + i64::from(value.hour), + i64::from(value.minute), + i64::from(second), + i64::from(value.nanosecond.unwrap_or_default() / NANOSECONDS_PER_MICROSECOND), + ) + .ok_or_else(|| invalid_datetime("the time does not exist")) +} + +fn datetime(value: &Datetime) -> Result { + let offset = value.offset.map(|offset| match offset { + Offset::Z => 0, + Offset::Custom { minutes } => i32::from(minutes) * SECONDS_PER_MINUTE, + }); + match (value.date, value.time) { + (Some(day), Some(clock)) => Ok(ConfigValue::DateTime(ConfigDateTime { + date: date(day)?, + time: time(clock)?, + utc_offset_seconds: offset, + })), + (Some(day), None) => date(day).map(ConfigValue::Date), + (None, Some(clock)) => time(clock).map(ConfigValue::Time), + (None, None) => Err(invalid_datetime("empty datetime")), + } +} + +/// A converted value and the earliest source offset of a key inside it, which orders tables. +type Positioned = (ConfigValue, usize); + +fn too_deep() -> ConfigError { + ConfigError::new( + ConfigErrorKind::Unsupported, + "TOML values nest deeper than the native limit", + ) +} + +fn key_position(key: Option<&Key>) -> usize { + key.and_then(Key::span) + .map_or(NO_POSITION, |span| span.start) +} + +/// A dictionary in `tomllib`'s order: keys in the order their first statement created them. +fn ordered_map(mut entries: Vec<(usize, String, ConfigValue)>) -> Positioned { + entries.sort_by_key(|(position, _, _)| *position); + let first = entries + .first() + .map_or(NO_POSITION, |(position, _, _)| *position); + let converted: Vec<(ConfigValue, ConfigValue)> = entries + .into_iter() + .map(|(_, key, value)| (ConfigValue::String(key), value)) + .collect(); + (ConfigValue::Map(converted), first) +} + +fn list(items: Vec) -> Positioned { + let first = items + .iter() + .map(|(_, position)| *position) + .min() + .unwrap_or(NO_POSITION); + let values: Vec = items.into_iter().map(|(value, _)| value).collect(); + (ConfigValue::List(values), first) +} + +fn array(values: &Array, depth: usize) -> Result { + values + .iter() + .map(|item| value(item, depth + 1)) + .collect::, _>>() + .map(list) +} + +fn inline_table(entries: &InlineTable, depth: usize) -> Result { + let mut converted: Vec<(usize, String, ConfigValue)> = Vec::new(); + for (key, item) in entries.iter() { + let (converted_value, inner) = value(item, depth + 1)?; + let position = key_position(entries.key(key)).min(inner); + converted.push((position, key.to_owned(), converted_value)); + } + Ok(ordered_map(converted)) +} + +fn value(item: &Value, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(too_deep()); + } + let scalar = |converted: ConfigValue| (converted, NO_POSITION); + match item { + Value::String(text) => Ok(scalar(ConfigValue::String(text.value().clone()))), + Value::Integer(number) => Ok(scalar(ConfigValue::Integer(*number.value()))), + Value::Float(number) => Ok(scalar(ConfigValue::Float(*number.value()))), + Value::Boolean(flag) => Ok(scalar(ConfigValue::Bool(*flag.value()))), + Value::Datetime(moment) => datetime(moment.value()).map(scalar), + Value::Array(values) => array(values, depth), + Value::InlineTable(entries) => inline_table(entries, depth), + } +} + +fn item(entry: &Item, depth: usize) -> Result, ConfigError> { + match entry { + Item::None => Ok(None), + Item::Value(scalar) => value(scalar, depth).map(Some), + Item::Table(entries) => table(entries, depth).map(Some), + Item::ArrayOfTables(tables) => tables + .iter() + .map(|entries| table(entries, depth + 1)) + .collect::, _>>() + .map(|tables| Some(list(tables))), + } +} + +fn table(entries: &Table, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(too_deep()); + } + let mut converted: Vec<(usize, String, ConfigValue)> = Vec::new(); + for (key, entry) in entries.iter() { + if let Some((converted_value, inner)) = item(entry, depth + 1)? { + let position = key_position(entries.key(key)).min(inner); + converted.push((position, key.to_owned(), converted_value)); + } + } + Ok(ordered_map(converted)) +} + +/// Parse `text` and convert its root table. +pub(crate) fn parse_document(text: &str) -> Result { + let document = Document::parse(text).map_err(|error| { + let offset = error.span().map_or(0, |span| span.start); + ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.message().to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + } + })?; + table(document.as_table(), 0).map(|(converted, _)| converted) +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/mod.rs b/crates/sqlbuild-config/src/toml/_helpers/mod.rs new file mode 100644 index 0000000000..cd76bd6968 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod conversion; +pub(crate) mod namespaces; +pub(crate) mod version; diff --git a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs new file mode 100644 index 0000000000..9961ccba4f --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs @@ -0,0 +1,344 @@ +//! `tomllib`'s namespace rules: which tables, dotted keys and arrays of tables may be (re)opened. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use std::collections::HashMap; +use toml_parser::decoder::Encoding; +use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; +use toml_parser::{ErrorSink, ParseError, Raw, Source, Span}; + +/// `Flags.FROZEN`: a value written with `=`, which later statements cannot extend. +const FROZEN: u8 = 1; +/// `Flags.EXPLICIT_NEST`: a table declared by a header or opened by a dotted key. +const EXPLICIT_NEST: u8 = 2; +/// The container depth the version check already enforces. +const MAX_CONTAINER_DEPTH: u32 = 80; + +/// One key of `tomllib`'s `Flags` tree. +#[derive(Default)] +struct FlagNode { + flags: u8, + recursive: u8, + nested: HashMap, +} + +/// `tomllib`'s `Flags`, keyed by table names without array indices. +#[derive(Default)] +struct Flags { + root: HashMap, + pending: Vec>, +} + +impl Flags { + fn is(&self, key: &[String], flag: u8) -> bool { + let Some((stem, parent)) = key.split_last() else { + return false; + }; + let mut container = &self.root; + for name in parent { + let Some(inner) = container.get(name) else { + return false; + }; + if inner.recursive & flag != 0 { + return true; + } + container = &inner.nested; + } + container + .get(stem) + .is_some_and(|node| (node.flags | node.recursive) & flag != 0) + } + + fn set(&mut self, key: &[String], flag: u8, recursive: bool) { + let Some((stem, parent)) = key.split_last() else { + return; + }; + let mut container = &mut self.root; + for name in parent { + container = &mut container.entry(name.clone()).or_default().nested; + } + let node = container.entry(stem.clone()).or_default(); + match recursive { + true => node.recursive |= flag, + false => node.flags |= flag, + } + } + + fn unset_all(&mut self, key: &[String]) { + let Some((stem, parent)) = key.split_last() else { + return; + }; + let mut container = &mut self.root; + for name in parent { + let Some(inner) = container.get_mut(name) else { + return; + }; + container = &mut inner.nested; + } + container.remove(stem); + } + + fn finalize_pending(&mut self) { + for key in std::mem::take(&mut self.pending) { + self.set(&key, EXPLICIT_NEST, false); + } + } +} + +/// What `tomllib`'s `NestedDict` holds behind one key. +enum Entry { + Table(HashMap), + Tables(Vec>), + Value, +} + +/// `tomllib`'s `NestedDict`: the tables and values created so far. +#[derive(Default)] +struct NestedDict { + root: HashMap, +} + +impl NestedDict { + /// `get_or_create_nest`; `None` where `tomllib` raises "There is no nest behind this key". + fn nest(&mut self, key: &[String], access_lists: bool) -> Option<&mut HashMap> { + let mut container = &mut self.root; + for name in key { + let entry = container + .entry(name.clone()) + .or_insert_with(|| Entry::Table(HashMap::new())); + container = match entry { + Entry::Table(table) => table, + Entry::Tables(tables) if access_lists => tables.last_mut()?, + _ => return None, + }; + } + Some(container) + } +} + +/// The keys and values of one inline table, checked as `tomllib`'s `parse_inline_table` does. +#[derive(Default)] +struct InlineTable { + data: NestedDict, + flags: Flags, + key: Vec, +} + +impl InlineTable { + fn insert(&mut self, container_value: bool) -> bool { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return false; + }; + if self.flags.is(&key, FROZEN) { + return false; + } + let Some(table) = self.data.nest(parent, false) else { + return false; + }; + if table.contains_key(stem) { + return false; + } + table.insert(stem.clone(), Entry::Value); + if container_value { + self.flags.set(&key, FROZEN, true); + } + true + } +} + +/// An open array or inline table inside a value. +enum Frame { + Array, + InlineTable(InlineTable), +} + +/// Replays `tomllib.loads`' bookkeeping over parser events and records the first rejection. +struct NamespaceReceiver<'text> { + source: Source<'text>, + data: NestedDict, + flags: Flags, + header: Vec, + key: Vec, + frames: Vec, + rejected: Option<(usize, &'static str)>, +} + +impl NamespaceReceiver<'_> { + fn reject(&mut self, span: Span, reason: &'static str) { + self.rejected.get_or_insert((span.start(), reason)); + } + + fn decode(&self, span: Span, encoding: Option) -> String { + let raw = self.source.get(span).map_or("", |raw| raw.as_str()); + let mut decoded = String::new(); + let mut ignored: Vec = Vec::new(); + Raw::new_unchecked(raw, encoding, span).decode_key(&mut decoded, &mut ignored); + decoded + } + + /// `key_value_rule` for a key-value pair under the current header. + fn key_value(&mut self, span: Span, container_value: bool) { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return; + }; + for length in 1..key.len() { + let container: Vec = [&self.header[..], &key[..length]].concat(); + if self.flags.is(&container, EXPLICIT_NEST) { + return self.reject(span, "Cannot redefine namespace"); + } + self.flags.pending.push(container); + } + let parent_key: Vec = [&self.header[..], parent].concat(); + if self.flags.is(&parent_key, FROZEN) { + return self.reject(span, "Cannot mutate immutable namespace"); + } + let Some(table) = self.data.nest(&parent_key, true) else { + return self.reject(span, "Cannot overwrite a value"); + }; + if table.contains_key(stem) { + return self.reject(span, "Cannot overwrite a value"); + } + table.insert(stem.clone(), Entry::Value); + if container_value { + let full_key: Vec = [&self.header[..], &key[..]].concat(); + self.flags.set(&full_key, FROZEN, true); + } + } + + /// `create_dict_rule` for a `[table]` header. + fn table(&mut self, span: Span) { + let key = std::mem::take(&mut self.key); + if self.flags.is(&key, EXPLICIT_NEST) || self.flags.is(&key, FROZEN) { + return self.reject(span, "Cannot declare a table twice"); + } + self.flags.set(&key, EXPLICIT_NEST, false); + if self.data.nest(&key, true).is_none() { + return self.reject(span, "Cannot overwrite a value"); + } + self.header = key; + } + + /// `create_list_rule` for an `[[array of tables]]` header. + fn array_of_tables(&mut self, span: Span) { + let key = std::mem::take(&mut self.key); + let Some((stem, parent)) = key.split_last() else { + return; + }; + if self.flags.is(&key, FROZEN) { + return self.reject(span, "Cannot mutate immutable namespace"); + } + self.flags.unset_all(&key); + self.flags.set(&key, EXPLICIT_NEST, false); + let Some(table) = self.data.nest(parent, true) else { + return self.reject(span, "Cannot overwrite a value"); + }; + match table.get_mut(stem) { + None => { + table.insert(stem.clone(), Entry::Tables(vec![HashMap::new()])); + } + Some(Entry::Tables(tables)) => tables.push(HashMap::new()), + Some(_) => return self.reject(span, "Cannot overwrite a value"), + } + self.header = key; + } + + /// A complete value: a scalar, or an array or inline table that just closed. + fn value(&mut self, span: Span, container_value: bool) { + match self.frames.last_mut() { + None => self.key_value(span, container_value), + Some(Frame::Array) => {} + Some(Frame::InlineTable(table)) => { + if !table.insert(container_value) { + self.reject(span, "Cannot redefine an inline table key"); + } + } + } + } + + fn open_header(&mut self) { + self.flags.finalize_pending(); + self.key.clear(); + } +} + +impl EventReceiver for NamespaceReceiver<'_> { + fn std_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.open_header(); + } + + fn std_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.table(span); + } + + fn array_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.open_header(); + } + + fn array_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.array_of_tables(span); + } + + fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.frames.push(Frame::InlineTable(InlineTable::default())); + true + } + + fn inline_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.frames.pop(); + self.value(span, true); + } + + fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.frames.push(Frame::Array); + true + } + + fn array_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.frames.pop(); + self.value(span, true); + } + + fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + let segment = self.decode(span, encoding); + match self.frames.last_mut() { + Some(Frame::InlineTable(table)) => table.key.push(segment), + Some(Frame::Array) => {} + None => self.key.push(segment), + } + } + + fn scalar(&mut self, span: Span, _encoding: Option, _error: &mut dyn ErrorSink) { + self.value(span, false); + } +} + +/// Reject what `tomllib` rejects as a redefined or immutable namespace. +pub(crate) fn reject_namespace_conflicts(text: &str) -> Result<(), ConfigError> { + let source = Source::new(text); + let tokens = source.lex().into_vec(); + let mut receiver = NamespaceReceiver { + source, + data: NestedDict::default(), + flags: Flags::default(), + header: Vec::new(), + key: Vec::new(), + frames: Vec::new(), + rejected: None, + }; + let mut errors: Vec = Vec::new(); + parse_document( + &tokens, + &mut RecursionGuard::new(&mut receiver, MAX_CONTAINER_DEPTH), + &mut errors, + ); + match receiver.rejected { + None => Ok(()), + Some((offset, reason)) => Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: reason.to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }), + } +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/version.rs b/crates/sqlbuild-config/src/toml/_helpers/version.rs new file mode 100644 index 0000000000..008be45309 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/version.rs @@ -0,0 +1,182 @@ +//! Find TOML 1.1 syntax, which the parser accepts and Python's TOML 1.0 `tomllib` rejects. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use toml_parser::decoder::Encoding; +use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; +use toml_parser::{ErrorSink, ParseError, Source, Span}; + +/// The container depth `toml_edit` accepts; deeper documents are left to Python. +const MAX_CONTAINER_DEPTH: u32 = 80; +/// The most dotted key segments read natively, which bounds the table nesting of a document. +const MAX_KEY_SEGMENTS: usize = 128; + +/// The containers open at the current event. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Container { + InlineTable, + Array, +} + +/// Records the first TOML 1.1-only construct. +struct VersionReceiver<'text> { + source: Source<'text>, + containers: Vec, + after_comma: bool, + key_segments: usize, + found: Option<(usize, &'static str)>, +} + +impl VersionReceiver<'_> { + fn in_inline_table(&self) -> bool { + self.containers.last() == Some(&Container::InlineTable) + } + + fn record(&mut self, span: Span, construct: &'static str) { + self.found.get_or_insert((span.start(), construct)); + } + + fn token(&mut self) { + self.after_comma = false; + } + + fn end_key(&mut self) { + self.token(); + self.key_segments = 0; + } + + fn line_break(&mut self, span: Span) { + if self.in_inline_table() { + self.record(span, "a newline or comment inside an inline table"); + } + } + + /// Escapes added by TOML 1.1: `\e` and `\xHH`. + fn check_escapes(&mut self, span: Span, encoding: Option) { + if !matches!( + encoding, + Some(Encoding::BasicString | Encoding::MlBasicString) + ) { + return; + } + let raw = self + .source + .get(span) + .map(|raw| raw.as_str()) + .unwrap_or_default(); + let mut escaped = false; + for character in raw.chars() { + if escaped && matches!(character, 'e' | 'x') { + self.record(span, "a \\e or \\x escape"); + } + escaped = !escaped && character == '\\'; + } + } +} + +impl EventReceiver for VersionReceiver<'_> { + fn std_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn array_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.end_key(); + self.containers.push(Container::InlineTable); + true + } + + fn inline_table_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { + if self.after_comma { + self.record(span, "a trailing comma in an inline table"); + } + self.token(); + self.containers.pop(); + } + + fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + self.token(); + self.containers.push(Container::Array); + true + } + + fn array_close(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.token(); + self.containers.pop(); + } + + fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + self.token(); + self.key_segments += 1; + if self.key_segments > MAX_KEY_SEGMENTS { + self.record(span, "a key with more segments than the native limit"); + } + self.check_escapes(span, encoding); + } + + fn key_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.token(); + } + + fn key_val_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.end_key(); + } + + fn scalar(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { + self.token(); + self.check_escapes(span, encoding); + } + + fn value_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + self.key_segments = 0; + self.after_comma = self.in_inline_table(); + } + + fn comment(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.line_break(span); + } + + fn newline(&mut self, span: Span, _error: &mut dyn ErrorSink) { + self.key_segments = 0; + self.line_break(span); + } +} + +/// Reject TOML 1.1-only syntax, syntax errors, and nesting beyond the native limits. +pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { + let source = Source::new(text); + let tokens = source.lex().into_vec(); + let mut receiver = VersionReceiver { + source, + containers: Vec::new(), + after_comma: false, + key_segments: 0, + found: None, + }; + let mut errors: Vec = Vec::new(); + parse_document( + &tokens, + &mut RecursionGuard::new(&mut receiver, MAX_CONTAINER_DEPTH), + &mut errors, + ); + if let Some(error) = errors.first() { + let offset = error.unexpected().map_or(0, |span| span.start()); + return Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.description().to_owned(), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }); + } + match receiver.found { + None => Ok(()), + Some((offset, construct)) => Err(ConfigError { + kind: ConfigErrorKind::Syntax, + message: format!("{construct} is TOML 1.1 syntax, which tomllib rejects"), + line: Some(text.get(..offset).unwrap_or(text).matches('\n').count() + 1), + column: None, + }), + } +} diff --git a/crates/sqlbuild-config/src/toml/main/load_toml.rs b/crates/sqlbuild-config/src/toml/main/load_toml.rs new file mode 100644 index 0000000000..9e7ab66184 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/load_toml.rs @@ -0,0 +1,19 @@ +//! Load a TOML document the way Python's `tomllib.loads` does. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; + +/// The table `tomllib.loads(text)` returns, rejecting TOML 1.1 syntax as `tomllib` does. +pub fn load_toml(text: &str) -> Result { + if text.starts_with(BYTE_ORDER_MARK) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "invalid statement: tomllib does not skip a byte order mark", + )); + } + let text = text.replace("\r\n", "\n"); + crate::toml::_helpers::version::reject_toml_1_1_syntax(&text)?; + crate::toml::_helpers::namespaces::reject_namespace_conflicts(&text)?; + crate::toml::_helpers::conversion::parse_document(&text) +} diff --git a/crates/sqlbuild-config/src/toml/main/mod.rs b/crates/sqlbuild-config/src/toml/main/mod.rs new file mode 100644 index 0000000000..04683b5110 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/mod.rs @@ -0,0 +1 @@ +pub mod load_toml; diff --git a/crates/sqlbuild-config/src/toml/mod.rs b/crates/sqlbuild-config/src/toml/mod.rs new file mode 100644 index 0000000000..668a17d0aa --- /dev/null +++ b/crates/sqlbuild-config/src/toml/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/toml/tests.rs b/crates/sqlbuild-config/src/toml/tests.rs new file mode 100644 index 0000000000..55b6a54c6d --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_load_toml.rs"] +mod load_toml; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/toml/tests/helpers.rs b/crates/sqlbuild-config/src/toml/tests/helpers.rs new file mode 100644 index 0000000000..d8cd6e7cdd --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/helpers.rs @@ -0,0 +1,11 @@ +use crate::models::ConfigValue; + +/// A table from string keys, in the given order. +pub(super) fn table(entries: Vec<(&str, ConfigValue)>) -> ConfigValue { + ConfigValue::Map( + entries + .into_iter() + .map(|(key, value)| (ConfigValue::String(key.to_owned()), value)) + .collect(), + ) +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs new file mode 100644 index 0000000000..078edffbd4 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs @@ -0,0 +1,330 @@ +use crate::errors::ConfigErrorKind; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::toml::main::load_toml::load_toml; +use crate::toml::tests::helpers::table; +use crate::toml::tests::test_types::{HostileTomlTestCase, LoadTomlTestCase}; + +#[test] +fn given_toml_documents_when_loading_then_values_match_tomllib() { + let test_cases = [ + LoadTomlTestCase { + description: "tables keep document order", + text: "z = 1\n[b]\nx = true\n[a]\ny = 'q'", + expected_value: Ok(ConfigValue::Map(vec![ + (ConfigValue::String("z".to_owned()), ConfigValue::Integer(1)), + ( + ConfigValue::String("b".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("x".to_owned()), + ConfigValue::Bool(true), + )]), + ), + ( + ConfigValue::String("a".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("y".to_owned()), + ConfigValue::String("q".to_owned()), + )]), + ), + ])), + }, + LoadTomlTestCase { + description: "implicit parent tables are created where first named", + text: "[a.b]\nx = 1\n[c]\n[a.d]\ny = 2", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("a".to_owned()), + ConfigValue::Map(vec![ + ( + ConfigValue::String("b".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("x".to_owned()), + ConfigValue::Integer(1), + )]), + ), + ( + ConfigValue::String("d".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("y".to_owned()), + ConfigValue::Integer(2), + )]), + ), + ]), + ), + ( + ConfigValue::String("c".to_owned()), + ConfigValue::Map(vec![]), + ), + ])), + }, + LoadTomlTestCase { + description: "dotted keys and arrays of tables", + text: "p.q = 1\n[[r]]\ns = 1.5\n[[r]]", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("p".to_owned()), + ConfigValue::Map(vec![( + ConfigValue::String("q".to_owned()), + ConfigValue::Integer(1), + )]), + ), + ( + ConfigValue::String("r".to_owned()), + ConfigValue::List(vec![ + ConfigValue::Map(vec![( + ConfigValue::String("s".to_owned()), + ConfigValue::Float(1.5), + )]), + ConfigValue::Map(vec![]), + ]), + ), + ])), + }, + LoadTomlTestCase { + description: "offset datetime keeps its offset", + text: "t = 1979-05-27T00:32:00.999999999-07:00", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("t".to_owned()), + ConfigValue::DateTime(ConfigDateTime { + date: ConfigDate { + year: 1979, + month: 5, + day: 27, + }, + time: ConfigTime { + hour: 0, + minute: 32, + second: 0, + microsecond: 999_999, + }, + utc_offset_seconds: Some(-25_200), + }), + )])), + }, + LoadTomlTestCase { + description: "local date and time", + text: "d = 1979-05-27\nt = 07:32:00", + expected_value: Ok(ConfigValue::Map(vec![ + ( + ConfigValue::String("d".to_owned()), + ConfigValue::Date(ConfigDate { + year: 1979, + month: 5, + day: 27, + }), + ), + ( + ConfigValue::String("t".to_owned()), + ConfigValue::Time(ConfigTime { + hour: 7, + minute: 32, + second: 0, + microsecond: 0, + }), + ), + ])), + }, + LoadTomlTestCase { + description: "escaped backslash before e is TOML 1.0", + text: "a = \"\\\\e\"", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("a".to_owned()), + ConfigValue::String("\\e".to_owned()), + )])), + }, + LoadTomlTestCase { + description: "multi-line array is TOML 1.0", + text: "a = [\n1,\n]", + expected_value: Ok(ConfigValue::Map(vec![( + ConfigValue::String("a".to_owned()), + ConfigValue::List(vec![ConfigValue::Integer(1)]), + )])), + }, + LoadTomlTestCase { + description: "trailing comma in an inline table is TOML 1.1", + text: "a = {x = 1,}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "newline in an inline table is TOML 1.1", + text: "a = {\nx = 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "escape e is TOML 1.1", + text: "a = \"\\e\"", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "escape x is TOML 1.1", + text: "a = \"\\x41\"", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "time without seconds is TOML 1.1", + text: "a = 07:32", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "leap second is rejected", + text: "a = 1990-12-31T23:59:60Z", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "impossible date is rejected", + text: "a = 2021-02-30", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "duplicate keys are rejected", + text: "a = 1\na = 2", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an implicit table declared later keeps the position where it was created", + text: "[p.a.config]\nx = 1\n[p.b]\ny = 2\n[p.a]\nz = 3", + expected_value: Ok(table(vec![( + "p", + table(vec![ + ( + "a", + table(vec![ + ("config", table(vec![("x", ConfigValue::Integer(1))])), + ("z", ConfigValue::Integer(3)), + ]), + ), + ("b", table(vec![("y", ConfigValue::Integer(2))])), + ]), + )])), + }, + LoadTomlTestCase { + description: "a dotted key creates its table before a later header", + text: "q.x = 1\n[r]\n[q.s]", + expected_value: Ok(table(vec![ + ( + "q", + table(vec![("x", ConfigValue::Integer(1)), ("s", table(vec![]))]), + ), + ("r", table(vec![])), + ])), + }, + LoadTomlTestCase { + description: "a dotted key cannot reach into an array of tables", + text: "[[a.b]]\n[a]\nb.c.d = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a two-segment dotted key cannot reach into an array of tables", + text: "[[a.b]]\n[a]\nb.c = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a dotted key cannot reopen a table declared by a header", + text: "[a.b]\n[a]\nb.c = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot redeclare a table created by a dotted key", + text: "a.b = 1\n[a]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot extend a table opened by dotted keys in a section", + text: "[a]\nb.c = 1\n[a.b]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "a header cannot extend an inline table", + text: "a = {b = 1}\n[a.c]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an inline table cannot be extended by a dotted key", + text: "x = {a = {b = 1}, a.c = 2}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "an array of tables cannot extend a static array", + text: "a = [1]\n[[a]]", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "each array of tables element reopens its own sub-tables", + text: "[[t]]\n[t.c]\nx = 1\n[[t]]\n[t.c]\ny = 2", + expected_value: Ok(table(vec![( + "t", + ConfigValue::List(vec![ + table(vec![("c", table(vec![("x", ConfigValue::Integer(1))]))]), + table(vec![("c", table(vec![("y", ConfigValue::Integer(2))]))]), + ]), + )])), + }, + LoadTomlTestCase { + description: "dotted keys may extend the tables they created", + text: "a.b.c = 1\na.b.d = 2", + expected_value: Ok(table(vec![( + "a", + table(vec![( + "b", + table(vec![ + ("c", ConfigValue::Integer(1)), + ("d", ConfigValue::Integer(2)), + ]), + )]), + )])), + }, + LoadTomlTestCase { + description: "a byte order mark is rejected as tomllib rejects it", + text: "\u{feff}a = 1", + expected_value: Err(ConfigErrorKind::Syntax), + }, + LoadTomlTestCase { + description: "CRLF line endings are normalized inside multi-line strings", + text: "a = \"\"\"x\r\ny\"\"\"", + expected_value: Ok(table(vec![("a", ConfigValue::String("x\ny".to_owned()))])), + }, + ]; + for test_case in test_cases { + let actual = load_toml(test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hostile_toml_when_loading_then_native_reader_rejects_without_exhausting_the_stack() { + let test_cases = [ + HostileTomlTestCase { + description: "ten thousand nested arrays", + text: format!("a = {}{}", "[".repeat(10_000), "]".repeat(10_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "ten thousand nested inline tables", + text: format!("a = {}{}", "{b = ".repeat(10_000), "}".repeat(10_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "a dotted key with a hundred thousand segments", + text: format!("a{} = 1", ".a".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileTomlTestCase { + description: "a table header with a hundred thousand segments", + text: format!("[a{}]", ".a".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + ]; + for test_case in test_cases { + let actual = load_toml(&test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, + Err(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs new file mode 100644 index 0000000000..cb0c0f42cf --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; + +pub(super) struct LoadTomlTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, +} + +pub(super) struct HostileTomlTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_error: ConfigErrorKind, +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs b/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs new file mode 100644 index 0000000000..856b203850 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/anchors.rs @@ -0,0 +1,87 @@ +//! Read node properties and indicators from the source, which parser events do not carry. + +/// No code character precedes the position. +const NO_CODE: usize = usize::MAX; +const TOKEN_BOUNDARIES: [char; 5] = ['[', ']', '{', '}', ',']; +/// The characters PyYAML's `scan_anchor` accepts right after an anchor name. +const ANCHOR_TERMINATORS: [char; 8] = ['?', ':', ',', ']', '}', '%', '@', '`']; + +/// For each char index, the end of the last code character before it, outside comments. +pub(crate) fn code_ends(chars: &[char]) -> Vec { + let mut ends: Vec = Vec::with_capacity(chars.len() + 1); + let mut last = NO_CODE; + let mut in_comment = false; + for (index, character) in chars.iter().enumerate() { + ends.push(last); + if *character == '\n' { + in_comment = false; + continue; + } + let line_start = index == 0 || chars[index - 1] == '\n'; + if *character == '#' && (line_start || chars[index - 1].is_whitespace()) { + in_comment = true; + } + if !in_comment && !character.is_whitespace() { + last = index + 1; + } + } + ends.push(last); + ends +} + +/// The end of the last code character before `end`, skipping whitespace and comments. +fn code_end_before(code_ends: &[usize], end: usize) -> Option { + let last = code_ends[end.min(code_ends.len() - 1)]; + (last != NO_CODE).then_some(last) +} + +/// The token that ends right before `end`, skipping whitespace and comments, with its start. +fn token_before(chars: &[char], code_ends: &[usize], end: usize) -> Option<(usize, String)> { + let token_end = code_end_before(code_ends, end)?; + let token_start = chars[..token_end] + .iter() + .rposition(|character| character.is_whitespace() || TOKEN_BOUNDARIES.contains(character)) + .map_or(0, |boundary| boundary + 1); + Some((token_start, chars[token_start..token_end].iter().collect())) +} + +/// Whether a standalone `?` explicit key indicator is the last code before char index `end`. +pub(crate) fn explicit_key_before(chars: &[char], code_ends: &[usize], end: usize) -> bool { + code_end_before(code_ends, end).is_some_and(|index| { + let indicator = index - 1; + let standalone = indicator == 0 + || matches!( + chars[indicator - 1], + ' ' | '\t' | '\n' | '\r' | '[' | '{' | ',' + ); + chars[indicator] == '?' && standalone + }) +} + +/// The anchor name in `token` when PyYAML's scanner accepts the character after it. +fn anchor_token(chars: &[char], token_start: usize, token: &str) -> Option { + let name = token.strip_prefix('&')?; + let following = chars.get(token_start + token.chars().count()).copied(); + let terminated = following.is_none_or(|character| { + character.is_whitespace() || ANCHOR_TERMINATORS.contains(&character) + }); + terminated.then(|| name.to_owned()) +} + +/// The name of the anchor property written before the node starting at char index `start`. +pub(crate) fn anchor_name(chars: &[char], code_ends: &[usize], start: usize) -> Option { + let (token_start, token) = token_before(chars, code_ends, start)?; + if token.starts_with('&') { + return anchor_token(chars, token_start, &token); + } + let (previous_start, previous) = token_before(chars, code_ends, token_start)?; + anchor_token(chars, previous_start, &previous) +} + +/// Whether PyYAML's scanner accepts `name` as an anchor: ASCII letters, digits, `-` and `_`. +pub(crate) fn is_python_anchor_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs b/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs new file mode 100644 index 0000000000..5a6a2b4602 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/block_scalars.rs @@ -0,0 +1,217 @@ +//! PyYAML's `scan_block_scalar`, which reads literal and folded scalars and chomps their tails. + +use crate::constants::BYTE_ORDER_MARK; +use crate::yaml::constants::{KEEP_INDICATOR, STRIP_INDICATOR}; + +const END: char = '\0'; +const FOLDED_INDICATOR: char = '>'; +const LINE_FEED: &str = "\n"; +const DIGITS: [char; 10] = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']; + +fn is_break(character: char) -> bool { + matches!(character, '\r' | '\n' | '\u{85}' | '\u{2028}' | '\u{2029}') +} + +fn is_blank(character: char) -> bool { + matches!(character, ' ' | '\t') +} + +/// The column of char index `index`, counted as PyYAML's reader counts it. +pub(crate) fn column_of(chars: &[char], index: usize) -> usize { + let line_start = chars[..index.min(chars.len())] + .iter() + .rposition(|character| is_break(*character)) + .map_or(0, |line_break| line_break + 1); + chars[line_start..index.min(chars.len())] + .iter() + .filter(|character| **character != BYTE_ORDER_MARK) + .count() +} + +/// PyYAML's scanner position over the document characters. +struct BlockScanner<'chars> { + chars: &'chars [char], + index: usize, + column: usize, +} + +impl BlockScanner<'_> { + fn peek(&self, offset: usize) -> char { + self.chars.get(self.index + offset).copied().unwrap_or(END) + } + + fn forward(&mut self, count: usize) { + for _ in 0..count { + let character = self.peek(0); + self.index += 1; + if matches!(character, '\n' | '\u{85}' | '\u{2028}' | '\u{2029}') + || (character == '\r' && self.peek(0) != '\n') + { + self.column = 0; + } else if character != BYTE_ORDER_MARK { + self.column += 1; + } + } + } + + fn line_break(&mut self) -> String { + match self.peek(0) { + '\r' if self.peek(1) == '\n' => { + self.forward(2); + "\n".to_owned() + } + '\r' | '\n' | '\u{85}' => { + self.forward(1); + "\n".to_owned() + } + separator @ ('\u{2028}' | '\u{2029}') => { + self.forward(1); + separator.to_string() + } + _ => String::new(), + } + } + + fn at_line_end(&self) -> bool { + self.peek(0) == END || is_break(self.peek(0)) + } + + /// An indentation digit: `Some(None)` when absent and `None` for the invalid `0`. + fn digit(&mut self) -> Option> { + let Some(increment) = DIGITS.iter().position(|digit| *digit == self.peek(0)) else { + return Some(None); + }; + self.forward(1); + (increment > 0).then_some(Some(increment)) + } + + fn chomping(&mut self) -> Option { + let character = self.peek(0); + let chomping = matches!(character, KEEP_INDICATOR | STRIP_INDICATOR); + chomping.then(|| { + self.forward(1); + character == KEEP_INDICATOR + }) + } + + /// `scan_block_scalar_indicators`: the chomping flag and the explicit indentation increment. + fn indicators(&mut self) -> Option<(Option, Option)> { + let indicators = match self.chomping() { + Some(chomping) => (Some(chomping), self.digit()?), + None => { + let increment = self.digit()?; + (self.chomping(), increment) + } + }; + (self.peek(0) == ' ' || self.at_line_end()).then_some(indicators) + } + + /// `scan_block_scalar_ignored_line`: spaces and a comment after the indicators. + fn ignored_line(&mut self) -> Option<()> { + while self.peek(0) == ' ' { + self.forward(1); + } + if self.peek(0) == '#' { + while !self.at_line_end() { + self.forward(1); + } + } + self.at_line_end().then(|| { + self.line_break(); + }) + } + + /// `scan_block_scalar_indentation`: leading empty lines and the deepest indentation among them. + fn indentation(&mut self) -> (String, usize) { + let mut breaks = String::new(); + let mut max_indent = 0; + while self.peek(0) == ' ' || is_break(self.peek(0)) { + if self.peek(0) == ' ' { + self.forward(1); + max_indent = max_indent.max(self.column); + } else { + breaks.push_str(&self.line_break()); + } + } + (breaks, max_indent) + } + + /// `scan_block_scalar_breaks`: empty lines up to the next line indented to `indent`. + fn breaks(&mut self, indent: usize) -> String { + let mut breaks = String::new(); + while self.column < indent && self.peek(0) == ' ' { + self.forward(1); + } + while is_break(self.peek(0)) { + breaks.push_str(&self.line_break()); + while self.column < indent && self.peek(0) == ' ' { + self.forward(1); + } + } + breaks + } + + fn line(&mut self) -> String { + let start = self.index; + let mut length = 0; + while !(self.peek(length) == END || is_break(self.peek(length))) { + length += 1; + } + self.forward(length); + self.chars[start..start + length].iter().collect() + } +} + +/// Read the block scalar whose indicator is at `indicator`, under a block collection at `parent_indent`. +pub(crate) fn scan_block_scalar( + chars: &[char], + indicator: usize, + parent_indent: Option, +) -> Option<(String, usize)> { + let folded = chars.get(indicator) == Some(&FOLDED_INDICATOR); + let mut scanner = BlockScanner { + chars, + index: indicator, + column: column_of(chars, indicator), + }; + scanner.forward(1); + let (chomping, increment) = scanner.indicators()?; + scanner.ignored_line()?; + let min_indent = parent_indent.map_or(1, |indent| indent + 1); + let (mut breaks, indent) = match increment { + None => { + let (breaks, max_indent) = scanner.indentation(); + (breaks, min_indent.max(max_indent)) + } + Some(increment) => { + let indent = min_indent + increment - 1; + (scanner.breaks(indent), indent) + } + }; + let mut value = String::new(); + let mut line_break = String::new(); + while scanner.column == indent && scanner.peek(0) != END { + value.push_str(&breaks); + let leading_non_space = !is_blank(scanner.peek(0)); + value.push_str(&scanner.line()); + line_break = scanner.line_break(); + breaks = scanner.breaks(indent); + if scanner.column != indent || scanner.peek(0) == END { + break; + } + let joins_with_space = + folded && line_break == LINE_FEED && leading_non_space && !is_blank(scanner.peek(0)); + match (joins_with_space, breaks.is_empty()) { + (true, true) => value.push(' '), + (true, false) => {} + (false, _) => value.push_str(&line_break), + } + } + if chomping != Some(false) { + value.push_str(&line_break); + } + if chomping == Some(true) { + value.push_str(&breaks); + } + Some((value, scanner.index)) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs new file mode 100644 index 0000000000..7d04aaf9d9 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -0,0 +1,475 @@ +//! Compose parser events into one node graph, as PyYAML's composer does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::yaml::_helpers::anchors::{ + anchor_name, code_ends, explicit_key_before, is_python_anchor_name, +}; +use crate::yaml::_helpers::block_scalars::{column_of, scan_block_scalar}; +use crate::yaml::_helpers::resolver::resolve_scalar; +use crate::yaml::constants::{ + FLOW_KEY_INDICATOR, MAP_TAG, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG, + PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, +}; +use crate::yaml::models::{ComposedDocument, Node, NodeContent}; +use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag}; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +type Properties<'input> = (usize, Option>); + +/// The block collection a node belongs to, which sets PyYAML's indentation for block scalars. +#[derive(Clone, Copy, Debug)] +enum Parent { + Document, + /// A block mapping whose key starts at this char index. + MappingAt(usize), + SequenceEntry, +} + +/// Where a node sits: its nesting depth, whether a flow collection encloses it, and whether it is a key. +#[derive(Clone, Copy, Debug)] +struct Placement { + depth: usize, + flow: bool, + key: bool, + parent: Parent, +} + +impl Placement { + fn child(self, flow: bool, key: bool, parent: Parent) -> Self { + Self { + depth: self.depth + 1, + flow: self.flow || flow, + key, + parent, + } + } +} + +/// Composition state of one document. +struct Composer<'input> { + parser: Parser<'input, StrInput<'input>>, + chars: Vec, + code_ends: Vec, + document: ComposedDocument, + anchors: HashMap, + anchor_names: HashSet, + open_anchors: HashSet, + previous_end: usize, + current_end: usize, +} + +fn syntax_error(error: &ScanError) -> ConfigError { + ConfigError { + kind: ConfigErrorKind::Syntax, + message: error.info().to_owned(), + line: Some(error.marker().line()), + column: Some(error.marker().col()), + } +} + +fn full_tag(tag: Option<&Cow<'_, Tag>>) -> Option { + tag.map(|tag| format!("{}{}", tag.handle, tag.suffix)) +} + +fn unsupported(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Unsupported, message) +} + +impl<'input> Composer<'input> { + fn next(&mut self) -> Result<(Event<'input>, Span), ConfigError> { + match self.parser.next_event() { + Some(Ok(event)) => { + self.previous_end = self.current_end; + self.current_end = event.1.end.index(); + Ok(event) + } + Some(Err(error)) => Err(syntax_error(&error)), + None => Err(ConfigError::new( + ConfigErrorKind::Syntax, + "unexpected end of stream", + )), + } + } + + fn push(&mut self, tag: String, content: NodeContent) -> usize { + self.document.nodes.push(Node { tag, content }); + self.document.nodes.len() - 1 + } + + /// Record an anchor before its node's children, rejecting what PyYAML's composer rejects. + fn open_anchor(&mut self, anchor_id: usize, span: Span) -> Result<(), ConfigError> { + if anchor_id == 0 { + return Ok(()); + } + let name = anchor_name(&self.chars, &self.code_ends, span.start.index()) + .ok_or_else(|| unsupported("anchor name could not be recovered"))?; + if !is_python_anchor_name(&name) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + format!("anchor {name:?} has characters PyYAML does not accept"), + )); + } + if !self.anchor_names.insert(name.clone()) { + return Err(ConfigError::new( + ConfigErrorKind::Construct, + format!("found duplicate anchor {name:?}"), + )); + } + self.open_anchors.insert(anchor_id); + Ok(()) + } + + fn close_anchor(&mut self, anchor_id: usize, node: usize) -> usize { + if anchor_id != 0 { + self.open_anchors.remove(&anchor_id); + self.anchors.insert(anchor_id, node); + } + node + } + + fn alias(&self, anchor_id: usize) -> Result { + if self.open_anchors.contains(&anchor_id) { + return Err(unsupported("recursive aliases are left to Python")); + } + self.anchors + .get(&anchor_id) + .copied() + .ok_or_else(|| ConfigError::new(ConfigErrorKind::Syntax, "found undefined alias")) + } + + fn char_at(&self, index: usize) -> Option { + self.chars.get(index).copied() + } + + /// Whether an implicit key, from its first property to its `:` indicator, covers a line break. + fn spans_lines(&self, span: Span) -> bool { + let key_end = span.end.index().min(self.chars.len()); + let indicator = (key_end..self.chars.len()) + .find(|index| !matches!(self.chars[*index], ' ' | '\t' | '\n' | '\r')) + .filter(|index| self.chars[*index] == ':'); + let end = indicator.unwrap_or(key_end); + let mut start = self.previous_end; + while start < end { + match self.chars[start] { + ' ' | '\t' | '\n' | '\r' | ',' | '[' | '{' => start += 1, + '#' => { + while start < end && !matches!(self.chars[start], '\n' | '\r') { + start += 1; + } + } + _ => break, + } + } + let explicit = self.chars.get(start) == Some(&'?') + && matches!( + self.char_at(start + 1), + None | Some(' ' | '\t' | '\n' | '\r') + ); + !explicit + && self.chars[start.min(end)..end] + .iter() + .any(|character| matches!(character, '\n' | '\r')) + } + + /// Whether the collection starting at `span` is written in flow style. + fn is_flow(&self, span: Span) -> bool { + span.end.index() > span.start.index() + && matches!(self.char_at(span.start.index()), Some('[' | '{')) + } + + /// Reject plain scalars PyYAML's scanner reads differently from YAML 1.2. + fn check_plain( + &self, + value: &str, + span: Span, + placement: Placement, + ) -> Result<(), ConfigError> { + let start = span.start.index(); + let first = self.char_at(start); + if placement.flow && first == Some(':') && !value.is_empty() { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar in a flow collection cannot start with ':'", + )); + } + let colon = (span.end.index()..self.chars.len()) + .find(|index| !matches!(self.chars[*index], ' ' | '\t')) + .unwrap_or(self.chars.len()); + let adjacent_value = self.char_at(colon) == Some(':') + && matches!(self.char_at(colon + 1), Some(',' | ']' | '}')); + if placement.flow && adjacent_value { + return Err(unsupported( + "LibYAML rejects ':' directly before a flow indicator", + )); + } + let explicit_key = explicit_key_before(&self.chars, &self.code_ends, start); + if placement.key && value.is_empty() && first == Some(':') && !explicit_key { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a mapping key cannot be empty without '?'", + )); + } + Ok(()) + } + + /// The block scalar indicator between the previous event and the scalar's content. + fn block_indicator(&self, span: Span) -> Option { + let mut index = self.previous_end; + let mut token_start = true; + while index <= span.start.index() { + let character = self.char_at(index)?; + match character { + '|' | '>' => return Some(index), + '#' if token_start => { + while !matches!(self.char_at(index), None | Some('\n' | '\r')) { + index += 1; + } + continue; + } + '!' | '&' if token_start => { + while !matches!(self.char_at(index), None | Some(' ' | '\n' | '\r')) { + index += 1; + } + continue; + } + _ => {} + } + token_start = matches!(character, ' ' | '\n' | '\r'); + index += 1; + } + None + } + + /// The column of the `-` that starts the sequence entry holding the indicator at `indicator`. + fn entry_column(&self, indicator: usize) -> Option { + let mut end = indicator; + loop { + while end > 0 && self.chars[end - 1] == ' ' { + end -= 1; + } + let start = self.chars[..end] + .iter() + .rposition(|character| matches!(character, ' ' | '\n' | '\r')) + .map_or(0, |boundary| boundary + 1); + let token: String = self.chars[start..end].iter().collect(); + if token.starts_with(['!', '&']) { + end = start; + continue; + } + return (token == SEQUENCE_ENTRY_TOKEN && end < indicator) + .then(|| column_of(&self.chars, start)); + } + } + + /// PyYAML's value for a literal or folded scalar, cross-checked against the parser's reading. + fn block_scalar( + &self, + parsed: &str, + span: Span, + placement: Placement, + ) -> Result { + let deferred = || unsupported("this block scalar is left to Python"); + if placement.flow || placement.key { + return Err(deferred()); + } + let indicator = self.block_indicator(span).ok_or_else(deferred)?; + let parent_indent = match placement.parent { + Parent::Document => None, + Parent::MappingAt(key_start) => Some(column_of(&self.chars, key_start)), + Parent::SequenceEntry => Some(self.entry_column(indicator).ok_or_else(deferred)?), + }; + let (value, end) = + scan_block_scalar(&self.chars, indicator, parent_indent).ok_or_else(deferred)?; + let parsed_end = span.end.index(); + let gap = &self.chars[end.min(parsed_end)..end.max(parsed_end).min(self.chars.len())]; + let same_end = gap + .iter() + .all(|character| matches!(character, ' ' | '\n' | '\r')); + let same_content = value.trim_end_matches('\n') == parsed.trim_end_matches('\n'); + (same_end && same_content) + .then_some(value) + .ok_or_else(deferred) + } + + fn compose_node( + &mut self, + event: Event<'input>, + span: Span, + placement: Placement, + ) -> Result { + if placement.depth > MAX_NESTING_DEPTH { + return Err(unsupported("nesting is deeper than the native limit")); + } + match event { + Event::Alias(anchor_id) => self.alias(anchor_id), + Event::Scalar(value, style, anchor_id, tag) => { + if style == ScalarStyle::Plain + && placement.flow + && value.contains(FLOW_KEY_INDICATOR) + { + return Err(unsupported( + "PyYAML ends a flow scalar at '?' and LibYAML does not", + )); + } + if style == ScalarStyle::Plain + && value.starts_with(PLAIN_FORBIDDEN_FIRST_CHARACTERS) + { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar cannot start with this indicator", + )); + } + let flow_value_indicator = placement.flow && value.starts_with(':'); + if style == ScalarStyle::Plain && flow_value_indicator { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "a plain scalar in a flow collection cannot start with ':'", + )); + } + if placement.key && placement.flow && self.spans_lines(span) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "an implicit key in a flow collection must fit on one line", + )); + } + if style == ScalarStyle::Plain && tag.is_none() && anchor_id == 0 { + self.check_plain(&value, span, placement)?; + } + let value = match style { + ScalarStyle::Literal | ScalarStyle::Folded => { + Cow::Owned(self.block_scalar(&value, span, placement)?) + } + _ => value, + }; + let non_specific = full_tag(tag.as_ref()).as_deref() == Some(NON_SPECIFIC_TAG); + if non_specific && value.is_empty() && style == ScalarStyle::Plain { + return Err(unsupported( + "LibYAML and PyYAML differ on an empty '!' scalar", + )); + } + self.open_anchor(anchor_id, span)?; + let tag = scalar_tag(&value, style, full_tag(tag.as_ref())); + let node = self.push(tag, NodeContent::Scalar(value.into_owned())); + Ok(self.close_anchor(anchor_id, node)) + } + Event::SequenceStart(anchor_id, tag) => { + self.compose_sequence((anchor_id, tag), span, placement) + } + Event::MappingStart(anchor_id, tag) => { + self.compose_mapping((anchor_id, tag), span, placement) + } + _ => Err(ConfigError::new(ConfigErrorKind::Syntax, "expected a node")), + } + } + + fn compose_sequence( + &mut self, + properties: Properties<'input>, + span: Span, + placement: Placement, + ) -> Result { + let (anchor_id, tag) = properties; + self.open_anchor(anchor_id, span)?; + let item_placement = placement.child(self.is_flow(span), false, Parent::SequenceEntry); + let mut items: Vec = Vec::new(); + loop { + let (event, item_span) = self.next()?; + if matches!(event, Event::SequenceEnd) { + break; + } + items.push(self.compose_node(event, item_span, item_placement)?); + } + let tag = collection_tag(full_tag(tag.as_ref()), SEQ_TAG); + let node = self.push(tag, NodeContent::Sequence(items)); + Ok(self.close_anchor(anchor_id, node)) + } + + fn compose_mapping( + &mut self, + properties: Properties<'input>, + span: Span, + placement: Placement, + ) -> Result { + let (anchor_id, tag) = properties; + self.open_anchor(anchor_id, span)?; + let flow = self.is_flow(span); + let mut entries: Vec<(usize, usize)> = Vec::new(); + loop { + let (event, key_span) = self.next()?; + if matches!(event, Event::MappingEnd) { + break; + } + let key_parent = Parent::MappingAt(key_span.start.index()); + let key = + self.compose_node(event, key_span, placement.child(flow, true, key_parent))?; + let (value_event, value_span) = self.next()?; + let value_placement = placement.child(flow, false, key_parent); + let value = self.compose_node(value_event, value_span, value_placement)?; + entries.push((key, value)); + } + let tag = collection_tag(full_tag(tag.as_ref()), MAP_TAG); + let node = self.push(tag, NodeContent::Mapping(entries)); + Ok(self.close_anchor(anchor_id, node)) + } + + /// Compose the only document of the stream; a second document is an error, as in PyYAML. + fn compose_stream(mut self) -> Result { + self.next()?; + let (event, _) = self.next()?; + if matches!(event, Event::StreamEnd) { + return Ok(self.document); + } + let (root_event, root_span) = self.next()?; + let root_placement = Placement { + depth: 0, + flow: false, + key: false, + parent: Parent::Document, + }; + let root = self.compose_node(root_event, root_span, root_placement)?; + self.document.root = Some(root); + self.next()?; + let (after, _) = self.next()?; + if !matches!(after, Event::StreamEnd) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "expected a single document in the stream", + )); + } + Ok(self.document) + } +} + +/// PyYAML's tag for a scalar: plain scalars and the `!` tag resolve implicitly. +fn scalar_tag(value: &str, style: ScalarStyle, tag: Option) -> String { + match tag { + Some(tag) if tag != NON_SPECIFIC_TAG => tag, + Some(_) => resolve_scalar(value, true).to_owned(), + None => resolve_scalar(value, style == ScalarStyle::Plain).to_owned(), + } +} + +fn collection_tag(tag: Option, default: &str) -> String { + match tag { + Some(tag) if tag != NON_SPECIFIC_TAG => tag, + _ => default.to_owned(), + } +} + +/// Compose the single document of `text` into a node graph. +pub(crate) fn compose(text: &str) -> Result { + let chars: Vec = text.chars().collect(); + Composer { + parser: Parser::new_from_str(text), + code_ends: code_ends(&chars), + chars, + document: ComposedDocument::default(), + anchors: HashMap::new(), + anchor_names: HashSet::new(), + open_anchors: HashSet::new(), + previous_end: 0, + current_end: 0, + } + .compose_stream() +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs new file mode 100644 index 0000000000..10832a2862 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/constructor.rs @@ -0,0 +1,274 @@ +//! Build values from a composed document, as PyYAML's `SafeConstructor` does. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::yaml::_helpers::keys::{is_hashable, key_identity}; +use crate::yaml::_helpers::numbers::{construct_float, construct_int}; +use crate::yaml::_helpers::timestamps::construct_timestamp; +use crate::yaml::constants::{ + BASE_EXPANSION_BUDGET, BOOL_TAG, BOOL_WORDS, EXPANSION_BUDGET_PER_CHARACTER, FLOAT_TAG, + INT_TAG, MAP_TAG, MAX_CONSTRUCTION_DEPTH, MERGE_TAG, NULL_TAG, PYTHON_ONLY_TAGS, SEQ_TAG, + STR_TAG, TIMESTAMP_TAG, VALUE_TAG, +}; +use crate::yaml::models::{ComposedDocument, Node, NodeContent}; +use std::collections::HashMap; +use std::collections::hash_map::Entry; + +/// One mapping entry after merge keys are flattened; `string_key` marks a `=` key read as text. +type FlatEntry = (usize, usize, bool); + +fn construct_error(message: String) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +fn construct_bool(text: &str) -> Result { + let lowered = text.to_lowercase(); + BOOL_WORDS + .iter() + .find(|(word, _)| *word == lowered) + .map(|(_, flag)| ConfigValue::Bool(*flag)) + .ok_or_else(|| construct_error(format!("{lowered:?} is not a YAML boolean"))) +} + +/// SafeConstructor's scalar constructors, by resolved tag. +fn construct_scalar(tag: &str, text: &str) -> Result { + match tag { + NULL_TAG => Ok(ConfigValue::Null), + BOOL_TAG => construct_bool(text), + INT_TAG => construct_int(text), + FLOAT_TAG => construct_float(text), + STR_TAG => Ok(ConfigValue::String(text.to_owned())), + TIMESTAMP_TAG => construct_timestamp(text.strip_suffix('\n').unwrap_or(text)), + _ if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( + ConfigErrorKind::Unsupported, + format!("values tagged {tag} are left to Python"), + )), + SEQ_TAG | MAP_TAG => Err(construct_error(format!( + "expected a collection node for {tag}" + ))), + _ => Err(construct_error(format!( + "could not determine a constructor for the tag {tag:?}" + ))), + } +} + +/// A constructed value and the number of values it holds, itself included. +type Sized = (ConfigValue, usize); + +/// Construction state; values of nodes reached through several aliases are built once. +struct Constructor<'document> { + document: &'document ComposedDocument, + shared: Vec, + built: HashMap, + budget: usize, +} + +fn deferred(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Unsupported, message) +} + +/// The nesting levels below a constructed value, which construction already bounds. +fn height(value: &ConfigValue) -> usize { + match value { + ConfigValue::List(items) => 1 + items.iter().map(height).max().unwrap_or_default(), + ConfigValue::Map(entries) => { + 1 + entries + .iter() + .map(|(key, item)| height(key).max(height(item))) + .max() + .unwrap_or_default() + } + _ => 0, + } +} + +/// Whether each node is reachable from more than one place, through aliases. +fn shared_nodes(document: &ComposedDocument) -> Vec { + let mut references = vec![0_usize; document.nodes.len()]; + for node in &document.nodes { + match &node.content { + NodeContent::Scalar(_) => {} + NodeContent::Sequence(items) => { + for item in items { + references[*item] += 1; + } + } + NodeContent::Mapping(entries) => { + for (key, value) in entries { + references[*key] += 1; + references[*value] += 1; + } + } + } + } + references.into_iter().map(|count| count > 1).collect() +} + +impl Constructor<'_> { + fn node(&self, id: usize) -> &Node { + &self.document.nodes[id] + } + + /// Spend part of the budget, which bounds the values built, copied and merged per document. + fn spend(&mut self, amount: usize) -> Result<(), ConfigError> { + self.budget = self + .budget + .checked_sub(amount) + .ok_or_else(|| deferred("aliases expand beyond the native size budget"))?; + Ok(()) + } + + fn construct(&mut self, id: usize, depth: usize) -> Result { + if depth > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("values nest deeper than the native limit")); + } + if let Some((value, size, height)) = self.built.get(&id) { + if depth + height > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("aliased values nest deeper than the native limit")); + } + let (value, size) = (value.clone(), *size); + self.spend(size)?; + return Ok((value, size)); + } + self.spend(1)?; + let built = self.construct_new(id, depth)?; + if self.shared[id] { + let (value, size) = &built; + self.built.insert(id, (value.clone(), *size, height(value))); + } + Ok(built) + } + + fn construct_new(&mut self, id: usize, depth: usize) -> Result { + let node = self.node(id); + match (&node.content, node.tag.as_str()) { + (NodeContent::Scalar(text), tag) => construct_scalar(tag, text).map(|value| (value, 1)), + (NodeContent::Sequence(items), SEQ_TAG) => { + let items = items.clone(); + let mut values: Vec = Vec::with_capacity(items.len()); + let mut size = 1; + for item in items { + let (value, item_size) = self.construct(item, depth + 1)?; + values.push(value); + size += item_size; + } + Ok((ConfigValue::List(values), size)) + } + (NodeContent::Mapping(_), MAP_TAG) => self.construct_mapping(id, depth), + (_, tag) if PYTHON_ONLY_TAGS.contains(&tag) => Err(ConfigError::new( + ConfigErrorKind::Unsupported, + format!("values tagged {tag} are left to Python"), + )), + (_, tag) => Err(construct_error(format!( + "could not construct a collection tagged {tag:?}" + ))), + } + } + + /// The entries of a mapping node; merge keys accept only mappings and lists of mappings. + fn entries(&self, id: usize) -> Result, ConfigError> { + match &self.node(id).content { + NodeContent::Mapping(entries) => Ok(entries.clone()), + _ => Err(construct_error( + "expected a mapping or list of mappings for merging".to_owned(), + )), + } + } + + /// The mappings a merge key's value contributes, in PyYAML's order. + fn merged(&mut self, value: usize, depth: usize) -> Result, ConfigError> { + let entries = match &self.node(value).content { + NodeContent::Mapping(_) => self.flatten(value, depth + 1)?, + NodeContent::Sequence(items) => { + let mut parts: Vec> = Vec::new(); + for item in items.clone() { + self.entries(item)?; + parts.push(self.flatten(item, depth + 1)?); + } + parts.into_iter().rev().flatten().collect() + } + NodeContent::Scalar(_) => { + return Err(construct_error( + "expected a mapping or list of mappings for merging".to_owned(), + )); + } + }; + self.spend(entries.len())?; + Ok(entries) + } + + /// PyYAML's `flatten_mapping`: merged entries first, then the mapping's own entries. + fn flatten(&mut self, id: usize, depth: usize) -> Result, ConfigError> { + if depth > MAX_CONSTRUCTION_DEPTH { + return Err(deferred("merge keys nest deeper than the native limit")); + } + let mut merged: Vec = Vec::new(); + let mut own: Vec = Vec::new(); + for (key, value) in self.entries(id)? { + match self.node(key).tag.as_str() { + MERGE_TAG => merged.extend(self.merged(value, depth)?), + VALUE_TAG => own.push((key, value, true)), + _ => own.push((key, value, false)), + } + } + merged.extend(own); + Ok(merged) + } + + fn construct_key( + &mut self, + key: usize, + string_key: bool, + depth: usize, + ) -> Result { + match (&self.node(key).content, string_key) { + (NodeContent::Scalar(text), true) => { + let text = text.clone(); + self.spend(1)?; + Ok((ConfigValue::String(text), 1)) + } + _ => self.construct(key, depth + 1), + } + } + + /// A Python dictionary: a repeated key keeps its first position and takes the last value. + fn construct_mapping(&mut self, id: usize, depth: usize) -> Result { + let mut entries: Vec<(ConfigValue, ConfigValue)> = Vec::new(); + let mut size = 1; + let mut positions: HashMap = HashMap::new(); + for (key, value, string_key) in self.flatten(id, depth)? { + let (key, key_size) = self.construct_key(key, string_key, depth)?; + if !is_hashable(&key) { + return Err(construct_error("found unhashable key".to_owned())); + } + let (value, value_size) = self.construct(value, depth + 1)?; + size += key_size + value_size; + match positions.entry(key_identity(&key)) { + Entry::Occupied(position) => entries[*position.get()].1 = value, + Entry::Vacant(position) => { + position.insert(entries.len()); + entries.push((key, value)); + } + } + } + Ok((ConfigValue::Map(entries), size)) + } +} + +/// The value of a composed document; an empty stream is `None` in Python, here `Null`. +pub(crate) fn construct_document( + document: &ComposedDocument, + text_length: usize, +) -> Result { + let Some(root) = document.root else { + return Ok(ConfigValue::Null); + }; + let mut constructor = Constructor { + document, + shared: shared_nodes(document), + built: HashMap::new(), + budget: BASE_EXPANSION_BUDGET + .saturating_add(text_length.saturating_mul(EXPANSION_BUDGET_PER_CHARACTER)), + }; + constructor.construct(root, 0).map(|(value, _)| value) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/keys.rs b/crates/sqlbuild-config/src/yaml/_helpers/keys.rs new file mode 100644 index 0000000000..b332e79444 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/keys.rs @@ -0,0 +1,63 @@ +//! Python dictionary key semantics for constructed YAML mappings. + +use crate::constants::FEBRUARY; +use crate::models::{ConfigDateTime, ConfigValue}; +use crate::yaml::constants::{SECONDS_PER_DAY, SECONDS_PER_MINUTE}; + +/// Whether Python can hash the value, so it can be a dictionary key. +pub(crate) fn is_hashable(value: &ConfigValue) -> bool { + !matches!(value, ConfigValue::List(_) | ConfigValue::Map(_)) +} + +/// The exact integer text of an integral float, or `None` for fractions and non-finite floats. +fn integral_float_text(value: f64) -> Option { + let integral = value.is_finite() && value.trunc() == value; + integral.then(|| format!("{:.0}", value + 0.0)) +} + +/// Numeric keys as exact integer text where Python would compare them as equal integers. +fn integer_text(value: &ConfigValue) -> Option { + match value { + ConfigValue::Bool(flag) => Some(u8::from(*flag).to_string()), + ConfigValue::Integer(number) => Some(number.to_string()), + ConfigValue::BigInteger(text) => Some(text.clone()), + ConfigValue::Float(number) => integral_float_text(*number), + _ => None, + } +} + +fn day_number(value: &ConfigDateTime) -> i64 { + let (year, month, day) = ( + i64::from(value.date.year), + i64::from(value.date.month), + i64::from(value.date.day), + ); + let shifted_year = if month <= FEBRUARY { year - 1 } else { year }; + let era_day = (153 * ((month + 9) % 12) + 2) / 5 + day - 1; + shifted_year * 365 + shifted_year / 4 - shifted_year / 100 + shifted_year / 400 + era_day +} + +fn instant(value: &ConfigDateTime) -> (i64, u32) { + let seconds = day_number(value) * i64::from(SECONDS_PER_DAY) + + i64::from(value.time.hour) * 3600 + + i64::from(value.time.minute) * i64::from(SECONDS_PER_MINUTE) + + i64::from(value.time.second) + - i64::from(value.utc_offset_seconds.unwrap_or_default()); + (seconds, value.time.microsecond) +} + +/// Text equal for two hashable keys exactly when Python's `==` holds, as `1 == 1.0 == True`. +pub(crate) fn key_identity(value: &ConfigValue) -> String { + if let Some(integer) = integer_text(value) { + return format!("int:{integer}"); + } + match value { + ConfigValue::String(text) => format!("str:{text}"), + ConfigValue::Float(number) if number.is_nan() => "float:nan".to_owned(), + ConfigValue::Float(number) => format!("float:{}", number.to_bits()), + ConfigValue::DateTime(moment) if moment.utc_offset_seconds.is_some() => { + format!("aware:{:?}", instant(moment)) + } + other => format!("{other:?}"), + } +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/mod.rs b/crates/sqlbuild-config/src/yaml/_helpers/mod.rs new file mode 100644 index 0000000000..6b6b26ca3e --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod anchors; +pub(crate) mod block_scalars; +pub(crate) mod composer; +pub(crate) mod constructor; +pub(crate) mod keys; +pub(crate) mod numbers; +pub(crate) mod patterns; +pub(crate) mod reader; +pub(crate) mod resolver; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs new file mode 100644 index 0000000000..d756cfdcfa --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/numbers.rs @@ -0,0 +1,196 @@ +//! PyYAML's `construct_yaml_int` and `construct_yaml_float` with Python's integer sizes. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::yaml::constants::{PYTHON_INT_MAX_STR_DIGITS, SEXAGESIMAL_BASE, ZERO_TEXT}; + +const LIMB_BASE: u64 = 1_000_000_000; +const LIMB_DIGITS: usize = 9; + +/// A non-negative arbitrary-precision integer in base 10^9 limbs, least significant first. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct Natural { + limbs: Vec, +} + +impl Natural { + fn multiply_add(self, factor: u32, addend: u32) -> Self { + let mut carry = u64::from(addend); + let mut limbs: Vec = self + .limbs + .into_iter() + .map(|limb| { + let total = u64::from(limb) * u64::from(factor) + carry; + carry = total / LIMB_BASE; + u32::try_from(total % LIMB_BASE).unwrap_or_default() + }) + .collect(); + while carry > 0 { + limbs.push(u32::try_from(carry % LIMB_BASE).unwrap_or_default()); + carry /= LIMB_BASE; + } + Self { limbs } + } + + fn add(self, other: &Self) -> Self { + let length = self.limbs.len().max(other.limbs.len()); + let mut carry = 0_u64; + let mut limbs: Vec = (0..length) + .map(|index| { + let total = u64::from(self.limbs.get(index).copied().unwrap_or_default()) + + u64::from(other.limbs.get(index).copied().unwrap_or_default()) + + carry; + carry = total / LIMB_BASE; + u32::try_from(total % LIMB_BASE).unwrap_or_default() + }) + .collect(); + if carry > 0 { + limbs.push(u32::try_from(carry).unwrap_or_default()); + } + Self { limbs } + } + + fn decimal(&self) -> String { + let mut limbs = self.limbs.iter().rev().skip_while(|limb| **limb == 0); + let Some(first) = limbs.next() else { + return "0".to_owned(); + }; + limbs.fold(first.to_string(), |text, limb| { + format!("{text}{limb:0LIMB_DIGITS$}") + }) + } +} + +fn construct_error(message: String) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +/// Python's `int(text, radix)` for the ASCII inputs PyYAML passes after removing underscores. +fn python_int(text: &str, radix: u32) -> Result<(bool, Natural), ConfigError> { + let trimmed = text.trim_matches(|character: char| character.is_ascii_whitespace()); + let (negative, unsigned) = match trimmed.as_bytes().first() { + Some(b'-') => (true, &trimmed[1..]), + Some(b'+') => (false, &trimmed[1..]), + _ => (false, trimmed), + }; + let prefix = match radix { + 2 => ["0b", "0B"], + 8 => ["0o", "0O"], + 16 => ["0x", "0X"], + _ => ["", ""], + }; + let digits = prefix + .iter() + .filter(|prefix| !prefix.is_empty()) + .find_map(|prefix| unsigned.strip_prefix(prefix)) + .unwrap_or(unsigned); + if digits.is_empty() { + return Err(construct_error(format!( + "invalid literal for int() with base {radix}: {text:?}" + ))); + } + let mut value = Natural::default(); + for character in digits.chars() { + let Some(digit) = character.to_digit(radix) else { + return Err(construct_error(format!( + "invalid literal for int() with base {radix}: {text:?}" + ))); + }; + value = value.multiply_add(radix, digit); + } + Ok((negative, value)) +} + +fn integer_value(negative: bool, value: &Natural) -> ConfigValue { + let decimal = value.decimal(); + let signed = if negative && decimal != ZERO_TEXT { + format!("-{decimal}") + } else { + decimal + }; + match signed.parse::() { + Ok(small) => ConfigValue::Integer(small), + Err(_) => ConfigValue::BigInteger(signed), + } +} + +fn split_sign(text: &str) -> Result<(bool, &str), ConfigError> { + match text.as_bytes().first() { + None => Err(construct_error("string index out of range".to_owned())), + Some(b'-') => Ok((true, &text[1..])), + Some(b'+') => Ok((false, &text[1..])), + Some(_) => Ok((false, text)), + } +} + +/// PyYAML's `construct_yaml_int`, including binary, octal, hexadecimal and base 60. +pub(crate) fn construct_int(text: &str) -> Result { + let cleaned = text.replace('_', ""); + let (negative, value) = split_sign(&cleaned)?; + if value.len() > PYTHON_INT_MAX_STR_DIGITS { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "integers longer than Python's string conversion limit are left to Python", + )); + } + let (inner_negative, magnitude) = if value == ZERO_TEXT { + (false, Natural::default()) + } else if let Some(binary) = value.strip_prefix("0b") { + python_int(binary, 2)? + } else if let Some(hexadecimal) = value.strip_prefix("0x") { + python_int(hexadecimal, 16)? + } else if value.starts_with('0') { + python_int(value, 8)? + } else if value.contains(':') { + value + .split(':') + .try_fold((false, Natural::default()), |(_, total), part| { + let (part_negative, digit) = python_int(part, 10)?; + if part_negative { + return Err(construct_error( + "negative sexagesimal digits are left to Python".to_owned(), + )); + } + Ok((false, total.multiply_add(SEXAGESIMAL_BASE, 0).add(&digit))) + })? + } else { + python_int(value, 10)? + }; + Ok(integer_value(negative != inner_negative, &magnitude)) +} + +/// Python's `float(text)` for the ASCII inputs PyYAML passes. +fn python_float(text: &str) -> Result { + text.trim_matches(|character: char| character.is_ascii_whitespace()) + .parse::() + .map_err(|_| construct_error(format!("could not convert string to float: {text:?}"))) +} + +/// Base-60 digits summed from the least significant one, in PyYAML's floating-point order. +fn sexagesimal_float(text: &str) -> Result { + let digits = text + .split(':') + .map(python_float) + .collect::, _>>()?; + let (total, _) = digits + .iter() + .rev() + .fold((0.0, 1.0), |(total, base), digit| { + (total + digit * base, base * f64::from(SEXAGESIMAL_BASE)) + }); + Ok(total) +} + +/// PyYAML's `construct_yaml_float`, including `.inf`, `.nan` and base 60. +pub(crate) fn construct_float(text: &str) -> Result { + let cleaned = text.replace('_', "").to_lowercase(); + let (negative, value) = split_sign(&cleaned)?; + let sign = if negative { -1.0 } else { 1.0 }; + let magnitude = match value { + ".inf" => f64::INFINITY, + ".nan" => return Ok(ConfigValue::Float(f64::NAN)), + _ if value.contains(':') => sexagesimal_float(value)?, + _ => python_float(value)?, + }; + Ok(ConfigValue::Float(sign * magnitude)) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs b/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs new file mode 100644 index 0000000000..d59ff2e3e5 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/patterns.rs @@ -0,0 +1,112 @@ +//! Anchored, hand-written equivalents of PyYAML's regular expressions. + +/// A cursor over ASCII bytes for hand-written, anchored versions of PyYAML's patterns. +#[derive(Clone, Debug)] +pub(crate) struct PatternCursor<'text> { + bytes: &'text [u8], + position: usize, +} + +impl<'text> PatternCursor<'text> { + pub(crate) fn new(text: &'text str) -> Self { + Self { + bytes: text.as_bytes(), + position: 0, + } + } + + /// Whether one alternative consumes all of `text`, like an anchored regular expression. + pub(crate) fn matches_any( + text: &str, + alternatives: &[fn(&mut PatternCursor<'_>) -> bool], + ) -> bool { + alternatives.iter().any(|alternative| { + let mut cursor = PatternCursor::new(text); + alternative(&mut cursor) && cursor.at_end() + }) + } + + pub(crate) fn peek(&self) -> Option { + self.bytes.get(self.position).copied() + } + + pub(crate) fn position(&self) -> usize { + self.position + } + + pub(crate) fn rewind(&mut self, position: usize) { + self.position = position; + } + + /// Exactly one byte accepted by `accept`. + pub(crate) fn one(&mut self, accept: impl Fn(u8) -> bool) -> bool { + let matched = self.peek().is_some_and(accept); + self.position += usize::from(matched); + matched + } + + /// Zero or more bytes accepted by `accept`; always succeeds. + pub(crate) fn many(&mut self, accept: impl Fn(u8) -> bool) -> bool { + while self.one(&accept) {} + true + } + + /// One or more bytes accepted by `accept`. + pub(crate) fn some(&mut self, accept: impl Fn(u8) -> bool) -> bool { + self.one(&accept) && self.many(accept) + } + + /// Up to `limit` bytes accepted by `accept`, returning how many were taken. + pub(crate) fn up_to(&mut self, limit: usize, accept: impl Fn(u8) -> bool) -> usize { + let start = self.position; + while self.position - start < limit && self.one(&accept) {} + self.position - start + } + + pub(crate) fn literal(&mut self, text: &str) -> bool { + let matched = self.bytes[self.position..].starts_with(text.as_bytes()); + self.position += usize::from(matched) * text.len(); + matched + } + + pub(crate) fn any_literal(&mut self, texts: &[&str]) -> bool { + texts.iter().any(|text| self.literal(text)) + } + + /// An optional `-` or `+`; always succeeds. + pub(crate) fn sign(&mut self) -> bool { + self.one(|byte| byte == b'-' || byte == b'+'); + true + } + + /// The optional exponent `[eE][-+][0-9]+`; always succeeds. + pub(crate) fn exponent(&mut self) -> bool { + let start = self.position; + let complete = self.one(|byte| byte == b'e' || byte == b'E') + && self.one(|byte| byte == b'-' || byte == b'+') + && self.some(|byte| byte.is_ascii_digit()); + if !complete { + self.position = start; + } + true + } + + /// `(?::[0-5]?[0-9])+`, one or more sexagesimal groups. + pub(crate) fn sexagesimal_groups(&mut self) -> bool { + let mut groups = 0; + while self.peek() == Some(b':') { + let width = match &self.bytes[self.position + 1..] { + [b'0'..=b'5', b'0'..=b'9', ..] => 2, + [b'0'..=b'9', ..] => 1, + _ => return false, + }; + self.position += 1 + width; + groups += 1; + } + groups > 0 + } + + pub(crate) fn at_end(&self) -> bool { + self.position == self.bytes.len() + } +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/reader.rs b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs new file mode 100644 index 0000000000..9c568414be --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/reader.rs @@ -0,0 +1,73 @@ +//! PyYAML's reader checks, applied before parsing. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::yaml::constants::{ + DIRECTIVE_INDICATOR, DOCUMENT_END_MARKER, LOADER_DEPENDENT_CHARACTERS, PYTHON_ONLY_LINE_BREAKS, +}; + +/// Whether PyYAML's reader accepts `character` (`Reader.NON_PRINTABLE` does not match it). +fn is_printable(character: char) -> bool { + matches!( + character, + '\t' | '\n' | '\r' | ' '..='~' | '\u{85}' | '\u{a0}'..='\u{d7ff}' | '\u{e000}'..='\u{fffd}' + ) || character >= '\u{10000}' +} + +/// Whether a line holds only whitespace or a comment. +fn is_blank_or_comment(line: &str) -> bool { + let content = line.trim_start_matches([' ', '\r']); + content.is_empty() || content.starts_with('#') +} + +/// Whether the first token of the stream is a `...` document end marker, which PyYAML rejects. +fn starts_with_document_end(text: &str) -> bool { + text.split('\n') + .find(|line| !is_blank_or_comment(line)) + .and_then(|line| line.strip_prefix(DOCUMENT_END_MARKER)) + .is_some_and(|rest| rest.is_empty() || rest.starts_with([' ', '\r'])) +} + +/// The text PyYAML's scanner reads: it skips one byte order mark at the very start. +pub(crate) fn without_byte_order_mark(text: &str) -> &str { + text.strip_prefix(BYTE_ORDER_MARK).unwrap_or(text) +} + +/// Reject what PyYAML's reader rejects, and defer characters PyYAML and LibYAML read differently. +pub(crate) fn check_characters(text: &str) -> Result<(), ConfigError> { + if let Some(character) = text.chars().find(|character| !is_printable(*character)) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + format!("unacceptable character {:#x}", u32::from(character)), + )); + } + if text.contains(PYTHON_ONLY_LINE_BREAKS) { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "NEL, LINE SEPARATOR and PARAGRAPH SEPARATOR are YAML 1.1 line breaks", + )); + } + if text.contains(LOADER_DEPENDENT_CHARACTERS) { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "tabs and inner byte order marks are read differently by PyYAML and LibYAML", + )); + } + let directive = text.starts_with(DIRECTIVE_INDICATOR) + || text + .split(['\n', '\r']) + .any(|line| line.starts_with(DIRECTIVE_INDICATOR)); + if directive { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "directives are left to Python", + )); + } + if starts_with_document_end(text) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "expected the node content, but found a document end marker", + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs b/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs new file mode 100644 index 0000000000..b136c739af --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/resolver.rs @@ -0,0 +1,113 @@ +//! PyYAML's YAML 1.1 implicit resolvers for plain scalars, without regular expressions. + +use crate::yaml::_helpers::patterns::PatternCursor; + +/// A plain-scalar matcher, the tag it resolves to, and the first characters PyYAML tries it for. +type ScalarResolver = (fn(&str) -> bool, &'static str, &'static str); +use crate::yaml::_helpers::timestamps::matches_implicit_timestamp; +use crate::yaml::constants::{ + BOOL_TAG, BOOL_WORDS, FLOAT_TAG, INT_TAG, MERGE_KEY, MERGE_TAG, NULL_TAG, NULL_WORDS, STR_TAG, + TIMESTAMP_TAG, VALUE_KEY, VALUE_TAG, +}; + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +fn is_digit_or_underscore(byte: u8) -> bool { + byte.is_ascii_digit() || byte == b'_' +} + +fn matches_float(text: &str) -> bool { + let alternatives: [fn(&mut PatternCursor<'_>) -> bool; 5] = [ + |cursor| { + cursor.sign() + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.literal(".") + && cursor.many(is_digit_or_underscore) + && cursor.exponent() + }, + |cursor| { + cursor.literal(".") + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.exponent() + }, + |cursor| { + cursor.sign() + && cursor.one(is_digit) + && cursor.many(is_digit_or_underscore) + && cursor.sexagesimal_groups() + && cursor.literal(".") + && cursor.many(is_digit_or_underscore) + }, + |cursor| cursor.sign() && cursor.any_literal(&[".inf", ".Inf", ".INF"]), + |cursor| cursor.any_literal(&[".nan", ".NaN", ".NAN"]), + ]; + PatternCursor::matches_any(text, &alternatives) +} + +fn matches_int(text: &str) -> bool { + let alternatives: [fn(&mut PatternCursor<'_>) -> bool; 6] = [ + |cursor| { + cursor.sign() + && cursor.literal("0b") + && cursor.some(|byte| matches!(byte, b'0' | b'1' | b'_')) + }, + |cursor| { + cursor.sign() + && cursor.literal("0") + && cursor.some(|byte| matches!(byte, b'0'..=b'7' | b'_')) + }, + |cursor| cursor.sign() && cursor.literal("0"), + |cursor| { + cursor.sign() + && cursor.one(|byte| matches!(byte, b'1'..=b'9')) + && cursor.many(is_digit_or_underscore) + }, + |cursor| { + cursor.sign() + && cursor.literal("0x") + && cursor.some(|byte| byte.is_ascii_hexdigit() || byte == b'_') + }, + |cursor| { + cursor.sign() + && cursor.one(|byte| matches!(byte, b'1'..=b'9')) + && cursor.many(is_digit_or_underscore) + && cursor.sexagesimal_groups() + }, + ]; + PatternCursor::matches_any(text, &alternatives) +} + +/// The tag PyYAML's `Resolver.resolve` gives a scalar; `implicit` is true for plain scalars. +pub(crate) fn resolve_scalar(value: &str, implicit: bool) -> &'static str { + if !implicit { + return STR_TAG; + } + let candidate = value.strip_suffix('\n').unwrap_or(value); + let resolvers: [ScalarResolver; 7] = [ + ( + |text| BOOL_WORDS.iter().any(|(word, _)| *word == text), + BOOL_TAG, + "yYnNtTfFoO", + ), + (matches_float, FLOAT_TAG, "-+0123456789."), + (matches_int, INT_TAG, "-+0123456789"), + (|text| text == MERGE_KEY, MERGE_TAG, "<"), + (|text| NULL_WORDS.contains(&text), NULL_TAG, "~nN"), + (matches_implicit_timestamp, TIMESTAMP_TAG, "0123456789"), + (|text| text == VALUE_KEY, VALUE_TAG, "="), + ]; + let first = value.chars().next(); + resolvers + .iter() + .filter(|(_, tag, first_characters)| { + first.map_or(*tag == NULL_TAG, |character| { + first_characters.contains(character) + }) + }) + .find(|(matches, _, _)| matches(candidate)) + .map_or(STR_TAG, |(_, tag, _)| tag) +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs b/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs new file mode 100644 index 0000000000..845c840049 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/_helpers/timestamps.rs @@ -0,0 +1,187 @@ +//! YAML 1.1 timestamps: PyYAML's implicit resolver pattern and `construct_yaml_timestamp`. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::yaml::_helpers::patterns::PatternCursor; +use crate::yaml::constants::{ + MICROSECOND_DIGITS, NEGATIVE_SIGN, SECONDS_PER_DAY, SECONDS_PER_MINUTE, +}; + +/// The `tz_sign`, `tz_hour` and `tz_minute` groups; `Z` has an empty sign. +type ZoneParts<'text> = (&'text str, &'text str, Option<&'text str>); + +/// The groups of PyYAML's `timestamp_regexp`. +#[derive(Default)] +struct TimestampParts<'text> { + year: &'text str, + month: &'text str, + day: &'text str, + time: Option<(&'text str, &'text str, &'text str)>, + fraction: Option<&'text str>, + zone: Option>, +} + +fn is_digit(byte: u8) -> bool { + byte.is_ascii_digit() +} + +fn is_blank(byte: u8) -> bool { + byte == b' ' || byte == b'\t' +} + +/// Matches PyYAML's timestamp pattern over one text. +struct TimestampScanner<'text> { + text: &'text str, + cursor: PatternCursor<'text>, +} + +impl<'text> TimestampScanner<'text> { + /// Between `bounds.0` and `bounds.1` digits. + fn digits(&mut self, bounds: (usize, usize)) -> Option<&'text str> { + let start = self.cursor.position(); + let taken = self.cursor.up_to(bounds.1, is_digit); + (taken >= bounds.0).then(|| &self.text[start..start + taken]) + } + + /// `-` followed by between `bounds.0` and `bounds.1` digits. + fn dashed(&mut self, bounds: (usize, usize)) -> Option<&'text str> { + self.cursor.literal("-").then(|| self.digits(bounds))? + } + + /// `:` followed by two digits. + fn sexagesimal(&mut self) -> Option<&'text str> { + self.cursor.literal(":").then(|| self.digits((2, 2)))? + } + + fn time(&mut self) -> Option<(&'text str, &'text str, &'text str)> { + let separated = self.cursor.one(|byte| byte == b'T' || byte == b't') + || self.cursor.up_to(usize::MAX, is_blank) > 0; + if !separated { + return None; + } + Some(( + self.digits((1, 2))?, + self.sexagesimal()?, + self.sexagesimal()?, + )) + } + + fn zone(&mut self) -> Option> { + let start = self.cursor.position(); + self.cursor.many(is_blank); + if self.cursor.literal("Z") { + return Some(("", "", None)); + } + let sign_start = self.cursor.position(); + let zone = if self.cursor.one(|byte| byte == b'-' || byte == b'+') { + self.digits((1, 2)).map(|hour| { + ( + &self.text[sign_start..=sign_start], + hour, + self.sexagesimal(), + ) + }) + } else { + None + }; + if zone.is_none() { + self.cursor.rewind(start); + } + zone + } + + /// Match the constructor pattern; `strict_date` requires the resolver's two-digit date. + fn parse(mut self, strict_date: bool) -> Option> { + let date_digits = if strict_date { (2, 2) } else { (1, 2) }; + let mut parts = TimestampParts { + year: self.digits((4, 4))?, + month: self.dashed(date_digits)?, + day: self.dashed(date_digits)?, + ..TimestampParts::default() + }; + let time_start = self.cursor.position(); + parts.time = self.time(); + if parts.time.is_none() { + self.cursor.rewind(time_start); + return self.cursor.at_end().then_some(parts); + } + if self.cursor.literal(".") { + let start = self.cursor.position(); + self.cursor.many(is_digit); + parts.fraction = Some(&self.text[start..self.cursor.position()]); + } + parts.zone = self.zone(); + self.cursor.at_end().then_some(parts) + } +} + +fn parse_parts(text: &str, strict_date: bool) -> Option> { + TimestampScanner { + text, + cursor: PatternCursor::new(text), + } + .parse(strict_date) +} + +/// Whether PyYAML's implicit timestamp resolver matches a plain scalar. +pub(crate) fn matches_implicit_timestamp(text: &str) -> bool { + match parse_parts(text, false) { + Some(parts) if parts.time.is_none() => parse_parts(text, true).is_some(), + Some(_) => true, + None => false, + } +} + +fn number(text: &str) -> i64 { + text.bytes() + .fold(0, |total, digit| total * 10 + i64::from(digit - b'0')) +} + +fn invalid(message: &str) -> ConfigError { + ConfigError::new(ConfigErrorKind::Construct, message) +} + +fn microseconds(fraction: Option<&str>) -> i64 { + let fraction = fraction.unwrap_or_default(); + let truncated = &fraction[..fraction.len().min(MICROSECOND_DIGITS)]; + number(&format!("{truncated:0>) -> Result, ConfigError> { + let Some((sign, hour, minute)) = zone else { + return Ok(None); + }; + let minutes = number(hour) * 60 + minute.map_or(0, number); + let seconds = + i32::try_from(minutes).map_err(|_| invalid("offset is out of range"))? * SECONDS_PER_MINUTE; + if seconds >= SECONDS_PER_DAY { + return Err(invalid("offset must be strictly between -24 and 24 hours")); + } + Ok(Some(if sign == NEGATIVE_SIGN { + -seconds + } else { + seconds + })) +} + +/// PyYAML's `construct_yaml_timestamp`: a date, or a naive or aware datetime. +pub(crate) fn construct_timestamp(text: &str) -> Result { + let parts = parse_parts(text, false).ok_or_else(|| invalid("not a timestamp"))?; + let date = ConfigDate::checked(number(parts.year), number(parts.month), number(parts.day)) + .ok_or_else(|| invalid("the date does not exist"))?; + let Some((hour, minute, second)) = parts.time else { + return Ok(ConfigValue::Date(date)); + }; + let time = ConfigTime::checked( + number(hour), + number(minute), + number(second), + microseconds(parts.fraction), + ) + .ok_or_else(|| invalid("the time does not exist"))?; + Ok(ConfigValue::DateTime(ConfigDateTime { + date, + time, + utc_offset_seconds: utc_offset(parts.zone)?, + })) +} diff --git a/crates/sqlbuild-config/src/yaml/constants.rs b/crates/sqlbuild-config/src/yaml/constants.rs new file mode 100644 index 0000000000..2a5b16581e --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/constants.rs @@ -0,0 +1,62 @@ +//! YAML tags SafeLoader resolves and constructs. + +pub(crate) const NULL_TAG: &str = "tag:yaml.org,2002:null"; +pub(crate) const BOOL_TAG: &str = "tag:yaml.org,2002:bool"; +pub(crate) const INT_TAG: &str = "tag:yaml.org,2002:int"; +pub(crate) const FLOAT_TAG: &str = "tag:yaml.org,2002:float"; +pub(crate) const STR_TAG: &str = "tag:yaml.org,2002:str"; +pub(crate) const TIMESTAMP_TAG: &str = "tag:yaml.org,2002:timestamp"; +pub(crate) const MERGE_TAG: &str = "tag:yaml.org,2002:merge"; +pub(crate) const VALUE_TAG: &str = "tag:yaml.org,2002:value"; +pub(crate) const SEQ_TAG: &str = "tag:yaml.org,2002:seq"; +pub(crate) const MAP_TAG: &str = "tag:yaml.org,2002:map"; +pub(crate) const NON_SPECIFIC_TAG: &str = "!"; +pub(crate) const PYTHON_ONLY_TAGS: [&str; 4] = [ + "tag:yaml.org,2002:binary", + "tag:yaml.org,2002:set", + "tag:yaml.org,2002:omap", + "tag:yaml.org,2002:pairs", +]; +pub(crate) const MAX_NESTING_DEPTH: usize = 256; +pub(crate) const PYTHON_ONLY_LINE_BREAKS: [char; 3] = ['\u{85}', '\u{2028}', '\u{2029}']; +pub(crate) const BOOL_WORDS: [(&str, bool); 18] = [ + ("yes", true), + ("Yes", true), + ("YES", true), + ("no", false), + ("No", false), + ("NO", false), + ("true", true), + ("True", true), + ("TRUE", true), + ("false", false), + ("False", false), + ("FALSE", false), + ("on", true), + ("On", true), + ("ON", true), + ("off", false), + ("Off", false), + ("OFF", false), +]; +pub(crate) const NULL_WORDS: [&str; 5] = ["", "~", "null", "Null", "NULL"]; +pub(crate) const SEXAGESIMAL_BASE: u32 = 60; +pub(crate) const MICROSECOND_DIGITS: usize = 6; +pub(crate) const SECONDS_PER_MINUTE: i32 = 60; +pub(crate) const SECONDS_PER_DAY: i32 = 86_400; +pub(crate) const MERGE_KEY: &str = "<<"; +pub(crate) const VALUE_KEY: &str = "="; +pub(crate) const NEGATIVE_SIGN: &str = "-"; +pub(crate) const ZERO_TEXT: &str = "0"; +pub(crate) const DOCUMENT_END_MARKER: &str = "..."; +pub(crate) const LOADER_DEPENDENT_CHARACTERS: [char; 2] = ['\t', crate::constants::BYTE_ORDER_MARK]; +pub(crate) const MAX_CONSTRUCTION_DEPTH: usize = 256; +pub(crate) const BASE_EXPANSION_BUDGET: usize = 100_000; +pub(crate) const EXPANSION_BUDGET_PER_CHARACTER: usize = 16; +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; +pub(crate) const FLOW_KEY_INDICATOR: char = '?'; +pub(crate) const KEEP_INDICATOR: char = '+'; +pub(crate) const STRIP_INDICATOR: char = '-'; +pub(crate) const SEQUENCE_ENTRY_TOKEN: &str = "-"; +pub(crate) const DIRECTIVE_INDICATOR: char = '%'; +pub(crate) const PLAIN_FORBIDDEN_FIRST_CHARACTERS: [char; 5] = ['|', '>', '%', '@', '`']; diff --git a/crates/sqlbuild-config/src/yaml/main/mod.rs b/crates/sqlbuild-config/src/yaml/main/mod.rs new file mode 100644 index 0000000000..ed61b475fd --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/main/mod.rs @@ -0,0 +1 @@ +pub mod safe_load; diff --git a/crates/sqlbuild-config/src/yaml/main/safe_load.rs b/crates/sqlbuild-config/src/yaml/main/safe_load.rs new file mode 100644 index 0000000000..cb226e4f9c --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/main/safe_load.rs @@ -0,0 +1,12 @@ +//! Load one YAML document the way PyYAML's `yaml.safe_load` does. + +use crate::errors::ConfigError; +use crate::models::ConfigValue; + +/// The value `yaml.safe_load(text)` returns, with YAML 1.1 implicit typing and merge keys. +pub fn safe_load(text: &str) -> Result { + let text = crate::yaml::_helpers::reader::without_byte_order_mark(text); + crate::yaml::_helpers::reader::check_characters(text)?; + let document = crate::yaml::_helpers::composer::compose(text)?; + crate::yaml::_helpers::constructor::construct_document(&document, text.chars().count()) +} diff --git a/crates/sqlbuild-config/src/yaml/mod.rs b/crates/sqlbuild-config/src/yaml/mod.rs new file mode 100644 index 0000000000..748414121c --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/yaml/models.rs b/crates/sqlbuild-config/src/yaml/models.rs new file mode 100644 index 0000000000..01ade04ca4 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/models.rs @@ -0,0 +1,23 @@ +//! The composed node graph of one YAML document, before construction. + +/// The content of one node; aliases share node ids. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum NodeContent { + Scalar(String), + Sequence(Vec), + Mapping(Vec<(usize, usize)>), +} + +/// One node with its resolved tag, as PyYAML's composer produces it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Node { + pub(crate) tag: String, + pub(crate) content: NodeContent, +} + +/// A composed document; `root` is `None` for an empty stream. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct ComposedDocument { + pub(crate) nodes: Vec, + pub(crate) root: Option, +} diff --git a/crates/sqlbuild-config/src/yaml/tests.rs b/crates/sqlbuild-config/src/yaml/tests.rs new file mode 100644 index 0000000000..b1224a8c91 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_safe_load.rs"] +mod safe_load; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/yaml/tests/helpers.rs b/crates/sqlbuild-config/src/yaml/tests/helpers.rs new file mode 100644 index 0000000000..13097546b3 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/helpers.rs @@ -0,0 +1,73 @@ +use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; + +pub(super) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} + +/// A one-entry mapping `{"a": value}`. +pub(super) fn under_a(value: ConfigValue) -> ConfigValue { + ConfigValue::Map(vec![(text("a"), value)]) +} + +pub(super) fn mapping(entries: Vec<(ConfigValue, ConfigValue)>) -> ConfigValue { + ConfigValue::Map(entries) +} + +pub(super) fn datetime(parts: [u32; 7], utc_offset_seconds: Option) -> ConfigValue { + ConfigValue::DateTime(ConfigDateTime { + date: date_parts(parts[0], parts[1], parts[2]), + time: ConfigTime { + hour: u8::try_from(parts[3]).unwrap_or_default(), + minute: u8::try_from(parts[4]).unwrap_or_default(), + second: u8::try_from(parts[5]).unwrap_or_default(), + microsecond: parts[6], + }, + utc_offset_seconds, + }) +} + +pub(super) fn date(year: u32, month: u32, day: u32) -> ConfigValue { + ConfigValue::Date(date_parts(year, month, day)) +} + +fn date_parts(year: u32, month: u32, day: u32) -> ConfigDate { + ConfigDate { + year: u16::try_from(year).unwrap_or_default(), + month: u8::try_from(month).unwrap_or_default(), + day: u8::try_from(day).unwrap_or_default(), + } +} + +/// `a0` holds a list, and every later key nests the previous one through an alias. +pub(super) fn alias_chain(links: usize) -> String { + (1..links).fold("a0: &a0 [x]\n".to_owned(), |text, link| { + format!("{text}a{link}: &a{link} [*a{}]\n", link - 1) + }) +} + +/// Each level lists the previous level ten times through aliases. +pub(super) fn billion_laughs(levels: usize) -> String { + (1..levels).fold("a0: &a0 \"lol\"\n".to_owned(), |text, level| { + let alias = format!("*a{}", level - 1); + format!( + "{text}a{level}: &a{level} [{}]\n", + vec![alias; 10].join(", ") + ) + }) +} + +/// Each level merges the previous level twice, doubling the merged entries. +pub(super) fn merge_chain(levels: usize) -> String { + (1..levels).fold("m0: &m0 {k0: 1}\n".to_owned(), |text, level| { + let previous = level - 1; + format!("{text}m{level}: &m{level} {{<<: [*m{previous}, *m{previous}], k{level}: 1}}\n") + }) +} + +/// Each anchor wraps the previous one in `depth` flow sequences. +pub(super) fn deep_anchor_chain(links: usize, depth: usize) -> String { + let (open, close) = ("[".repeat(depth), "]".repeat(depth)); + (1..links).fold(format!("a0: &a0 {open}x{close}\n"), |text, link| { + format!("{text}a{link}: &a{link} {open}*a{}{close}\n", link - 1) + }) +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs b/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs new file mode 100644 index 0000000000..ba3ea3bec4 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/test_safe_load.rs @@ -0,0 +1,438 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; +use crate::yaml::main::safe_load::safe_load; +use crate::yaml::tests::helpers::{ + alias_chain, billion_laughs, date, datetime, deep_anchor_chain, mapping, merge_chain, text, + under_a, +}; +use crate::yaml::tests::test_types::{HostileYamlTestCase, SafeLoadTestCase}; + +#[test] +fn given_yaml_documents_when_loading_then_values_match_pyyaml_safe_load() { + let test_cases = [ + SafeLoadTestCase { + description: "YAML 1.1 booleans", + text: "a: yes", + expected_value: Ok(under_a(ConfigValue::Bool(true))), + }, + SafeLoadTestCase { + description: "mixed-case boolean is text", + text: "a: yEs", + expected_value: Ok(under_a(text("yEs"))), + }, + SafeLoadTestCase { + description: "off is false", + text: "a: Off", + expected_value: Ok(under_a(ConfigValue::Bool(false))), + }, + SafeLoadTestCase { + description: "octal integer", + text: "a: 017", + expected_value: Ok(under_a(ConfigValue::Integer(15))), + }, + SafeLoadTestCase { + description: "0o is not YAML 1.1 octal", + text: "a: 0o17", + expected_value: Ok(under_a(text("0o17"))), + }, + SafeLoadTestCase { + description: "binary integer", + text: "a: 0b101", + expected_value: Ok(under_a(ConfigValue::Integer(5))), + }, + SafeLoadTestCase { + description: "hexadecimal integer", + text: "a: 0x1F", + expected_value: Ok(under_a(ConfigValue::Integer(31))), + }, + SafeLoadTestCase { + description: "underscores in integers", + text: "a: 1_000", + expected_value: Ok(under_a(ConfigValue::Integer(1000))), + }, + SafeLoadTestCase { + description: "sexagesimal integer", + text: "a: 190:20:30", + expected_value: Ok(under_a(ConfigValue::Integer(685_230))), + }, + SafeLoadTestCase { + description: "sexagesimal float", + text: "a: -190:20:30.5", + expected_value: Ok(under_a(ConfigValue::Float(-685_230.5))), + }, + SafeLoadTestCase { + description: "eight is not octal", + text: "a: 08", + expected_value: Ok(under_a(text("08"))), + }, + SafeLoadTestCase { + description: "arbitrary precision integer", + text: "a: 99999999999999999999999", + expected_value: Ok(under_a(ConfigValue::BigInteger( + "99999999999999999999999".to_owned(), + ))), + }, + SafeLoadTestCase { + description: "exponent needs a dot and a sign", + text: "a: 1e5", + expected_value: Ok(under_a(text("1e5"))), + }, + SafeLoadTestCase { + description: "float with signed exponent", + text: "a: 1.0e+5", + expected_value: Ok(under_a(ConfigValue::Float(100_000.0))), + }, + SafeLoadTestCase { + description: "float with trailing dot", + text: "a: 5.", + expected_value: Ok(under_a(ConfigValue::Float(5.0))), + }, + SafeLoadTestCase { + description: "signed leading-dot float is text", + text: "a: +.5", + expected_value: Ok(under_a(text("+.5"))), + }, + SafeLoadTestCase { + description: "negative infinity", + text: "a: -.Inf", + expected_value: Ok(under_a(ConfigValue::Float(f64::NEG_INFINITY))), + }, + SafeLoadTestCase { + description: "tilde is null", + text: "a: ~", + expected_value: Ok(under_a(ConfigValue::Null)), + }, + SafeLoadTestCase { + description: "empty value is null", + text: "a:", + expected_value: Ok(under_a(ConfigValue::Null)), + }, + SafeLoadTestCase { + description: "quoted values stay text", + text: "a: 'yes'", + expected_value: Ok(under_a(text("yes"))), + }, + SafeLoadTestCase { + description: "non-specific tag resolves quoted scalars", + text: "a: ! '123'", + expected_value: Ok(under_a(ConfigValue::Integer(123))), + }, + SafeLoadTestCase { + description: "explicit string tag", + text: "a: !!str 1", + expected_value: Ok(under_a(text("1"))), + }, + SafeLoadTestCase { + description: "explicit integer tag parses hexadecimal text", + text: "a: !!int '0x10'", + expected_value: Ok(under_a(ConfigValue::Integer(16))), + }, + SafeLoadTestCase { + description: "date", + text: "a: 2001-12-14", + expected_value: Ok(under_a(date(2001, 12, 14))), + }, + SafeLoadTestCase { + description: "single-digit month is text", + text: "a: 2001-1-4", + expected_value: Ok(under_a(text("2001-1-4"))), + }, + SafeLoadTestCase { + description: "aware timestamp", + text: "a: 2001-12-14t21:59:43.10-05:00", + expected_value: Ok(under_a(datetime( + [2001, 12, 14, 21, 59, 43, 100_000], + Some(-18_000), + ))), + }, + SafeLoadTestCase { + description: "spaced timestamp with short offset", + text: "a: 2001-12-14 21:59:43.10 -5", + expected_value: Ok(under_a(datetime( + [2001, 12, 14, 21, 59, 43, 100_000], + Some(-18_000), + ))), + }, + SafeLoadTestCase { + description: "naive timestamp truncates to microseconds", + text: "a: 2001-12-14 21:59:43.1234567", + expected_value: Ok(under_a(datetime([2001, 12, 14, 21, 59, 43, 123_456], None))), + }, + SafeLoadTestCase { + description: "merge keys put merged entries first", + text: "base: &b {x: 1, y: 2}\nother:\n y: 3\n <<: *b", + expected_value: Ok(mapping(vec![ + ( + text("base"), + mapping(vec![ + (text("x"), ConfigValue::Integer(1)), + (text("y"), ConfigValue::Integer(2)), + ]), + ), + ( + text("other"), + mapping(vec![ + (text("x"), ConfigValue::Integer(1)), + (text("y"), ConfigValue::Integer(3)), + ]), + ), + ])), + }, + SafeLoadTestCase { + description: "merge list prefers earlier mappings", + text: "a:\n <<: [{k: 1}, {k: 2, j: 3}]", + expected_value: Ok(under_a(mapping(vec![ + (text("k"), ConfigValue::Integer(1)), + (text("j"), ConfigValue::Integer(3)), + ]))), + }, + SafeLoadTestCase { + description: "equal numeric keys collapse to the first key", + text: "{1: a, 1.0: b, true: c}", + expected_value: Ok(mapping(vec![(ConfigValue::Integer(1), text("c"))])), + }, + SafeLoadTestCase { + description: "equals sign key is text", + text: "=: 1", + expected_value: Ok(mapping(vec![(text("="), ConfigValue::Integer(1))])), + }, + SafeLoadTestCase { + description: "empty stream is null", + text: "", + expected_value: Ok(ConfigValue::Null), + }, + SafeLoadTestCase { + description: "duplicate anchors are rejected", + text: "a: &x 1\nb: &x 2", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "equals sign value has no constructor", + text: "a: =", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "merge value outside a key has no constructor", + text: "a: <<", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "impossible date is rejected", + text: "a: 2002-02-30", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "unhashable key is rejected", + text: "{[1, 2]: a}", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "several documents are rejected", + text: "--- 1\n--- 2", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "hexadecimal without digits is rejected", + text: "a: 0x_", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "unknown tags are rejected", + text: "a: !custom 1", + expected_value: Err(ConfigErrorKind::Construct), + }, + SafeLoadTestCase { + description: "sets are left to Python", + text: "a: !!set {x}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "control characters are rejected", + text: "a: \u{7}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "YAML 1.1 line separators are left to Python", + text: "a: b\u{2028}c", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "recursive aliases are left to Python", + text: "a: &x [*x]", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "literal scalar at the end of input has no final line break", + text: "a: |\n x", + expected_value: Ok(under_a(text("x"))), + }, + SafeLoadTestCase { + description: "folded scalar at the end of input has no final line break", + text: "a: >\n folded", + expected_value: Ok(under_a(text("folded"))), + }, + SafeLoadTestCase { + description: "sequence entry literal at the end of input", + text: "- |\n x", + expected_value: Ok(ConfigValue::List(vec![text("x")])), + }, + SafeLoadTestCase { + description: "trailing indentation without a line break is not content", + text: "a: |\n x\n ", + expected_value: Ok(under_a(text("x\n"))), + }, + SafeLoadTestCase { + description: "keep chomping counts only complete trailing lines", + text: "a: |+\n x\n\n ", + expected_value: Ok(under_a(text("x\n\n"))), + }, + SafeLoadTestCase { + description: "strip chomping drops trailing lines", + text: "a: |-\n x\n\n", + expected_value: Ok(under_a(text("x"))), + }, + SafeLoadTestCase { + description: "a literal of blank lines clips to empty text", + text: "a: |\n \n", + expected_value: Ok(under_a(text(""))), + }, + SafeLoadTestCase { + description: "more-indented trailing spaces are content", + text: "a: |\n x\n ", + expected_value: Ok(under_a(text("x\n "))), + }, + SafeLoadTestCase { + description: "explicit indentation under a nested sequence", + text: "a:\n - |2-\n x\n y\n", + expected_value: Ok(under_a(ConfigValue::List(vec![text(" x\ny")]))), + }, + SafeLoadTestCase { + description: "a leading byte order mark is skipped", + text: "\u{feff}a: 1", + expected_value: Ok(under_a(ConfigValue::Integer(1))), + }, + SafeLoadTestCase { + description: "the non-specific tag keeps a quoted line break as text", + text: "a: ! \"\\n\"", + expected_value: Ok(under_a(text("\n"))), + }, + SafeLoadTestCase { + description: "sexagesimal digits may have leading zeros", + text: "a: 1:05", + expected_value: Ok(under_a(ConfigValue::Integer(65))), + }, + SafeLoadTestCase { + description: "a lone document end marker is rejected", + text: "...", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a document end marker after comments is rejected", + text: "# c\n...", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a flow key spanning lines is rejected", + text: "{a\nb: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a flow key with its value indicator on the next line is rejected", + text: "{a\n: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "an anchor glued to a flow collection is left to Python", + text: "&a{}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "a plain scalar cannot start with a block indicator", + text: "{>-: 1}", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "a question mark ending a value is not an explicit key", + text: "a: b?\n: c", + expected_value: Err(ConfigErrorKind::Syntax), + }, + SafeLoadTestCase { + description: "question marks in flow scalars are left to Python", + text: "[a?b]", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "tabs are left to Python", + text: "a: x\ty", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "inner byte order marks are left to Python", + text: "a: x\u{feff}", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + SafeLoadTestCase { + description: "directives are left to Python", + text: "%YAML 1.1\n--- a", + expected_value: Err(ConfigErrorKind::Unsupported), + }, + ]; + for test_case in test_cases { + let actual = safe_load(test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hostile_yaml_when_loading_then_native_reader_defers_without_exhausting_resources() { + let test_cases = [ + HostileYamlTestCase { + description: "a ten thousand link alias chain", + text: alias_chain(10_000), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "billion laughs with twelve levels", + text: billion_laughs(12), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "merge keys doubling over thirty levels", + text: merge_chain(30), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "anchors wrapping deep flow nesting", + text: deep_anchor_chain(50, 200), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "a hundred thousand nested flow sequences", + text: format!("a: {}{}", "[".repeat(100_000), "]".repeat(100_000)), + expected_error: ConfigErrorKind::Syntax, + }, + HostileYamlTestCase { + description: "a hundred thousand nested block sequences", + text: "- ".repeat(100_000), + expected_error: ConfigErrorKind::Unsupported, + }, + HostileYamlTestCase { + description: "an integer beyond Python's string conversion limit", + text: format!("a: {}", "9".repeat(4_301)), + expected_error: ConfigErrorKind::Unsupported, + }, + ]; + for test_case in test_cases { + let actual = safe_load(&test_case.text).map_err(|error| error.kind); + assert_eq!( + actual, + Err(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_types.rs b/crates/sqlbuild-config/src/yaml/tests/test_types.rs new file mode 100644 index 0000000000..8e36a9ebb0 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::errors::ConfigErrorKind; +use crate::models::ConfigValue; + +pub(super) struct SafeLoadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, +} + +pub(super) struct HostileYamlTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_error: ConfigErrorKind, +} diff --git a/crates/sqlbuild-core/src/json/_helpers/emitter.rs b/crates/sqlbuild-core/src/json/_helpers/emitter.rs new file mode 100644 index 0000000000..809c66b1d5 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/emitter.rs @@ -0,0 +1,143 @@ +//! Serialize a [`JsonValue`] in the layout of one Python serializer. + +use crate::json::_helpers::floats::{orjson_text, python_repr}; +use crate::json::_helpers::strings::{StringEscaping, json_string}; +use crate::json::constants::{ + MAX_NESTING_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, +}; +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue}; + +/// The resolved layout of one serializer configuration. +struct Layout<'options> { + indent: Option<&'options str>, + item_separator: &'options str, + key_separator: &'options str, + sort_keys: bool, + escaping: StringEscaping, + orjson: bool, + allow_nan: bool, +} + +impl<'options> Layout<'options> { + fn new(dialect: &'options JsonDialect) -> Self { + match dialect { + JsonDialect::Stdlib(options) => Self { + indent: options.indent.as_deref(), + item_separator: &options.item_separator, + key_separator: &options.key_separator, + sort_keys: options.sort_keys, + escaping: if options.ensure_ascii { + StringEscaping::Ascii + } else { + StringEscaping::Unicode + }, + orjson: false, + allow_nan: options.allow_nan, + }, + JsonDialect::Orjson(options) => Self { + indent: options.indent_2.then_some(" "), + item_separator: ",", + key_separator: if options.indent_2 { ": " } else { ":" }, + sort_keys: options.sort_keys, + escaping: StringEscaping::Unicode, + orjson: true, + allow_nan: true, + }, + } + } + + fn newline(&self, level: usize) -> String { + self.indent + .map_or_else(String::new, |indent| format!("\n{}", indent.repeat(level))) + } + + fn integer(&self, value: &JsonInteger) -> Result { + let in_range = value + .to_i128() + .is_some_and(|number| (ORJSON_MIN_INTEGER..=ORJSON_MAX_INTEGER).contains(&number)); + if self.orjson && !in_range { + return Err(JsonEmitError::IntegerOutOfRange); + } + let digits = value.as_str().trim_start_matches('-').len(); + if digits > PYTHON_INT_MAX_STR_DIGITS { + return Err(JsonEmitError::IntegerTooLong); + } + Ok(value.as_str().to_owned()) + } + + fn float(&self, value: f64) -> Result { + match value { + _ if value.is_finite() && self.orjson => Ok(orjson_text(value)), + _ if value.is_finite() => Ok(python_repr(value)), + _ if self.orjson => Ok("null".to_owned()), + _ if !self.allow_nan => Err(JsonEmitError::NonFiniteFloat), + _ if value.is_nan() => Ok("NaN".to_owned()), + _ if value.is_sign_positive() => Ok("Infinity".to_owned()), + _ => Ok("-Infinity".to_owned()), + } + } + + fn value(&self, value: &JsonValue, level: usize) -> Result { + if level > MAX_NESTING_DEPTH { + return Err(JsonEmitError::NestingTooDeep); + } + match value { + JsonValue::Null => Ok("null".to_owned()), + JsonValue::Bool(flag) => Ok(flag.to_string()), + JsonValue::Integer(number) => self.integer(number), + JsonValue::Float(number) => self.float(*number), + JsonValue::String(text) => Ok(json_string(text, self.escaping)), + JsonValue::Array(items) => { + let members = items + .iter() + .map(|item| self.value(item, level + 1)) + .collect::, _>>()?; + Ok(self.container(('[', ']'), members, level)) + } + JsonValue::Object(entries) => self.object(entries, level), + } + } + + fn object( + &self, + entries: &[(String, JsonValue)], + level: usize, + ) -> Result { + let mut ordered: Vec<&(String, JsonValue)> = entries.iter().collect(); + if self.sort_keys { + ordered.sort_by(|left, right| left.0.cmp(&right.0)); + } + let members = ordered + .into_iter() + .map(|(key, item)| { + Ok(format!( + "{}{}{}", + json_string(key, self.escaping), + self.key_separator, + self.value(item, level + 1)? + )) + }) + .collect::, _>>()?; + Ok(self.container(('{', '}'), members, level)) + } + + fn container(&self, brackets: (char, char), members: Vec, level: usize) -> String { + if members.is_empty() { + return format!("{}{}", brackets.0, brackets.1); + } + let opening = self.newline(level + 1); + let separator = format!("{}{opening}", self.item_separator); + let closing = self.newline(level); + format!( + "{}{opening}{}{closing}{}", + brackets.0, + members.join(&separator), + brackets.1 + ) + } +} + +pub(crate) fn emit(value: &JsonValue, dialect: &JsonDialect) -> Result { + Layout::new(dialect).value(value, 0) +} diff --git a/crates/sqlbuild-core/src/json/_helpers/floats.rs b/crates/sqlbuild-core/src/json/_helpers/floats.rs new file mode 100644 index 0000000000..04ebd950a1 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/floats.rs @@ -0,0 +1,74 @@ +//! Shortest round-trip float text in Python's and orjson's layouts. + +use crate::json::constants::{ + ORJSON_EXPONENT_DIGITS, ORJSON_FIXED_EXPONENTS, PYTHON_EXPONENT_DIGITS, PYTHON_FIXED_EXPONENTS, +}; +use std::ops::RangeInclusive; + +fn scientific_parts(text: &str) -> (String, i32) { + let (mantissa, exponent) = text.split_once('e').unwrap_or((text, "0")); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + let (sign, magnitude) = exponent + .strip_prefix('-') + .map_or((1, exponent), |magnitude| (-1, magnitude)); + let exponent = magnitude + .bytes() + .fold(0_i32, |total, digit| total * 10 + i32::from(digit - b'0')); + (digits, sign * exponent) +} + +/// Shortest round-trip digits of `|value|`, ties rounded half to even as CPython does. +fn shortest_digits(value: f64) -> (String, i32) { + let (shortest, _) = scientific_parts(&format!("{:e}", value.abs())); + let precision = shortest.len().saturating_sub(1); + scientific_parts(&format!("{:.precision$e}", value.abs())) +} + +/// Fixed-point layout of `digits` with the decimal point after `point` digits. +fn fixed(digits: &str, point: i32) -> String { + match usize::try_from(point) { + Err(_) | Ok(0) => format!("0.{}{digits}", "0".repeat(point.unsigned_abs() as usize)), + Ok(point) if point >= digits.len() => { + format!("{digits}{}.0", "0".repeat(point - digits.len())) + } + Ok(point) => format!("{}.{}", &digits[..point], &digits[point..]), + } +} + +fn scientific(digits: &str, exponent: i32, exponent_digits: usize) -> String { + let (first, rest) = digits.split_at(1); + let mantissa = if rest.is_empty() { + first.to_owned() + } else { + format!("{first}.{rest}") + }; + let sign = if exponent.is_negative() { '-' } else { '+' }; + let magnitude = exponent.unsigned_abs(); + format!("{mantissa}e{sign}{magnitude:0exponent_digits$}") +} + +fn layout(value: f64, fixed_exponents: RangeInclusive, exponent_digits: usize) -> String { + let (digits, exponent) = shortest_digits(value); + let body = if fixed_exponents.contains(&exponent) { + fixed(&digits, exponent + 1) + } else { + scientific(&digits, exponent, exponent_digits) + }; + let sign = if value.is_sign_negative() { "-" } else { "" }; + format!("{sign}{body}") +} + +/// Python's `repr(float)`, including `nan`, `inf` and `-inf`. +pub(crate) fn python_repr(value: f64) -> String { + match value { + _ if value.is_nan() => "nan".to_owned(), + _ if value.is_infinite() && value.is_sign_positive() => "inf".to_owned(), + _ if value.is_infinite() => "-inf".to_owned(), + _ => layout(value, PYTHON_FIXED_EXPONENTS, PYTHON_EXPONENT_DIGITS), + } +} + +/// orjson's text for a finite float. +pub(crate) fn orjson_text(value: f64) -> String { + layout(value, ORJSON_FIXED_EXPONENTS, ORJSON_EXPONENT_DIGITS) +} diff --git a/crates/sqlbuild-core/src/json/_helpers/mod.rs b/crates/sqlbuild-core/src/json/_helpers/mod.rs new file mode 100644 index 0000000000..44402147f5 --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod emitter; +pub(crate) mod floats; +pub(crate) mod strings; diff --git a/crates/sqlbuild-core/src/json/_helpers/strings.rs b/crates/sqlbuild-core/src/json/_helpers/strings.rs new file mode 100644 index 0000000000..169c27da2e --- /dev/null +++ b/crates/sqlbuild-core/src/json/_helpers/strings.rs @@ -0,0 +1,50 @@ +//! JSON string escaping of `json.dumps` and orjson. + +use crate::json::constants::FIRST_PRINTABLE_ASCII; + +/// How a serializer escapes characters outside printable ASCII. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StringEscaping { + /// `ensure_ascii=True`: `\uXXXX`, with UTF-16 surrogate pairs above U+FFFF. + Ascii, + /// `ensure_ascii=False` and orjson: only control characters are escaped. + Unicode, +} + +fn unicode_escape(unit: u32) -> String { + format!("\\u{unit:04x}") +} + +fn escaped(character: char, escaping: StringEscaping) -> String { + match character { + '"' => "\\\"".to_owned(), + '\\' => "\\\\".to_owned(), + '\n' => "\\n".to_owned(), + '\r' => "\\r".to_owned(), + '\t' => "\\t".to_owned(), + '\u{8}' => "\\b".to_owned(), + '\u{c}' => "\\f".to_owned(), + ' '..='~' => character.to_string(), + _ if character < FIRST_PRINTABLE_ASCII => unicode_escape(u32::from(character)), + _ if escaping == StringEscaping::Unicode => character.to_string(), + _ => character + .encode_utf16(&mut [0_u16; 2]) + .iter() + .map(|unit| unicode_escape(u32::from(*unit))) + .collect(), + } +} + +/// `text` as a quoted JSON string. +pub(crate) fn json_string(text: &str, escaping: StringEscaping) -> String { + let mut output = String::with_capacity(text.len() + 2); + output.push('"'); + for character in text.chars() { + match character { + ' '..='~' if character != '"' && character != '\\' => output.push(character), + _ => output.push_str(&escaped(character, escaping)), + } + } + output.push('"'); + output +} diff --git a/crates/sqlbuild-core/src/json/constants.rs b/crates/sqlbuild-core/src/json/constants.rs new file mode 100644 index 0000000000..fab04a225b --- /dev/null +++ b/crates/sqlbuild-core/src/json/constants.rs @@ -0,0 +1,13 @@ +//! Serializer constants. + +pub(crate) const ORJSON_MIN_INTEGER: i128 = i64::MIN as i128; +pub(crate) const ORJSON_MAX_INTEGER: i128 = u64::MAX as i128; +pub(crate) const FIRST_PRINTABLE_ASCII: char = ' '; +pub(crate) const PYTHON_FIXED_EXPONENTS: std::ops::RangeInclusive = -4..=15; +pub(crate) const ORJSON_FIXED_EXPONENTS: std::ops::RangeInclusive = -5..=15; +pub(crate) const PYTHON_EXPONENT_DIGITS: usize = 2; +pub(crate) const ORJSON_EXPONENT_DIGITS: usize = 1; +/// CPython's default `sys.get_int_max_str_digits()`, beyond which `int.__repr__` raises. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; +/// Containers nested deeper than this are refused; orjson's own limit is 255 and json's is higher. +pub(crate) const MAX_NESTING_DEPTH: usize = 128; diff --git a/crates/sqlbuild-core/src/json/errors.rs b/crates/sqlbuild-core/src/json/errors.rs new file mode 100644 index 0000000000..0c731dec9a --- /dev/null +++ b/crates/sqlbuild-core/src/json/errors.rs @@ -0,0 +1,14 @@ +//! Values a Python serializer refuses. + +/// A value the chosen serializer refuses, named after the Python failure it reproduces. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JsonEmitError { + /// `json.dumps(..., allow_nan=False)` met `NaN` or an infinity and raised `ValueError`. + NonFiniteFloat, + /// orjson met an integer outside `[-2**63, 2**64 - 1]` and raised `JSONEncodeError`. + IntegerOutOfRange, + /// `json.dumps` met an integer with more digits than Python's string conversion limit. + IntegerTooLong, + /// The value nests deeper than the native emitter writes; callers fall back to Python. + NestingTooDeep, +} diff --git a/crates/sqlbuild-core/src/json/main/dumps.rs b/crates/sqlbuild-core/src/json/main/dumps.rs new file mode 100644 index 0000000000..ccf47da133 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/dumps.rs @@ -0,0 +1,9 @@ +//! Serialize JSON exactly as SQLBuild's Python serializers do. + +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonValue}; + +/// The text `json.dumps(value, ...)` or `orjson.dumps(value, ...).decode()` returns. +pub fn dumps(value: &JsonValue, dialect: &JsonDialect) -> Result { + crate::json::_helpers::emitter::emit(value, dialect) +} diff --git a/crates/sqlbuild-core/src/json/main/mod.rs b/crates/sqlbuild-core/src/json/main/mod.rs new file mode 100644 index 0000000000..ca5e541504 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/mod.rs @@ -0,0 +1 @@ +pub mod dumps; diff --git a/crates/sqlbuild-core/src/json/mod.rs b/crates/sqlbuild-core/src/json/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-core/src/json/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/json/models.rs b/crates/sqlbuild-core/src/json/models.rs new file mode 100644 index 0000000000..351ec48f0a --- /dev/null +++ b/crates/sqlbuild-core/src/json/models.rs @@ -0,0 +1,138 @@ +//! JSON values and the Python serializer configurations the emitter reproduces. + +/// A JSON value with Python's value domain; objects keep insertion order and unique keys. +#[derive(Clone, Debug, PartialEq)] +pub enum JsonValue { + Null, + Bool(bool), + Integer(JsonInteger), + Float(f64), + String(String), + Array(Vec), + Object(Vec<(String, JsonValue)>), +} + +/// An arbitrary-precision integer as canonical decimal text without leading zeros or `-0`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct JsonInteger { + decimal: String, +} + +/// Which Python serializer, with which options, to reproduce. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JsonDialect { + /// The standard library's `json.dumps`. + Stdlib(StdlibJsonOptions), + /// `orjson.dumps`. + Orjson(OrjsonOptions), +} + +/// Options of `json.dumps`, with the same meaning and defaults. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StdlibJsonOptions { + /// `indent`, as the string repeated once per nesting level (`indent=2` is two spaces). + pub indent: Option, + /// First element of `separators`. + pub item_separator: String, + /// Second element of `separators`. + pub key_separator: String, + pub ensure_ascii: bool, + pub sort_keys: bool, + pub allow_nan: bool, +} + +/// The `orjson` options SQLBuild uses; every other option is off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct OrjsonOptions { + /// `orjson.OPT_INDENT_2`. + pub indent_2: bool, + /// `orjson.OPT_SORT_KEYS`. + pub sort_keys: bool, +} + +impl StdlibJsonOptions { + /// The options of `json.dumps(value, indent=indent)`, with Python's default separators. + pub fn new(indent: Option) -> Self { + let item_separator = if indent.is_some() { "," } else { ", " }; + Self { + indent: indent.map(|width| " ".repeat(width)), + item_separator: item_separator.to_owned(), + key_separator: ": ".to_owned(), + ensure_ascii: true, + sort_keys: false, + allow_nan: true, + } + } + + /// The same options with `separators=(item_separator, key_separator)`. + pub fn with_separators(self, item_separator: &str, key_separator: &str) -> Self { + Self { + item_separator: item_separator.to_owned(), + key_separator: key_separator.to_owned(), + ..self + } + } +} + +impl JsonInteger { + /// Parse decimal text with an optional sign and leading zeros, as Python's `int()` does. + pub fn parse(text: &str) -> Option { + let (negative, digits) = match text.as_bytes().first()? { + b'-' => (true, &text[1..]), + b'+' => (false, &text[1..]), + _ => (false, text), + }; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let significant = digits.trim_start_matches('0'); + let decimal = match (significant.is_empty(), negative) { + (true, _) => "0".to_owned(), + (false, true) => format!("-{significant}"), + (false, false) => significant.to_owned(), + }; + Some(Self { decimal }) + } + + /// The canonical decimal text. + pub fn as_str(&self) -> &str { + &self.decimal + } + + /// The value as an `i128`, or `None` when it needs more than 128 bits. + pub fn to_i128(&self) -> Option { + let (sign, digits) = match self.decimal.strip_prefix('-') { + Some(digits) => (-1, digits), + None => (1, self.decimal.as_str()), + }; + digits.bytes().try_fold(0_i128, |total, digit| { + total + .checked_mul(10)? + .checked_add(sign * i128::from(digit - b'0')) + }) + } +} + +impl From for JsonInteger { + fn from(value: i64) -> Self { + Self { + decimal: value.to_string(), + } + } +} + +impl From for JsonInteger { + fn from(value: u64) -> Self { + Self { + decimal: value.to_string(), + } + } +} + +impl From for JsonInteger { + fn from(value: i128) -> Self { + Self { + decimal: value.to_string(), + } + } +} diff --git a/crates/sqlbuild-core/src/json/tests.rs b/crates/sqlbuild-core/src/json/tests.rs new file mode 100644 index 0000000000..49c3d52143 --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_dumps.rs"] +mod dumps; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/json/tests/helpers.rs b/crates/sqlbuild-core/src/json/tests/helpers.rs new file mode 100644 index 0000000000..25229c37ff --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/helpers.rs @@ -0,0 +1,57 @@ +use crate::json::models::{JsonInteger, JsonValue}; + +fn string(text: &str) -> JsonValue { + JsonValue::String(text.to_owned()) +} + +fn integer(text: &str) -> JsonValue { + JsonValue::Integer(JsonInteger::parse(text).expect("test integers are decimal")) +} + +/// A value exercising escapes, big integers, float layouts, empty containers and key order. +pub(super) fn sample_document(largest_integer: &str) -> JsonValue { + let floats = [0.1, -0.0, 1e16, 1e-5, 2.5e-7, 1e15, 100.0]; + JsonValue::Object(vec![ + ("name".to_owned(), string("café ☕ 😀")), + ("control".to_owned(), string("\u{0}\u{1f}\u{7f}\t\n\"\\/")), + ( + "ints".to_owned(), + JsonValue::Array(vec![ + integer("0"), + integer("-1"), + integer("9223372036854775808"), + integer(largest_integer), + ]), + ), + ( + "floats".to_owned(), + JsonValue::Array(floats.into_iter().map(JsonValue::Float).collect()), + ), + ( + "empty".to_owned(), + JsonValue::Array(vec![JsonValue::Object(vec![]), JsonValue::Array(vec![])]), + ), + ("z".to_owned(), JsonValue::Null), + ("a".to_owned(), JsonValue::Bool(true)), + ]) +} + +pub(super) fn non_finite_floats() -> JsonValue { + JsonValue::Array(vec![ + JsonValue::Float(f64::NAN), + JsonValue::Float(f64::INFINITY), + JsonValue::Float(f64::NEG_INFINITY), + ]) +} + +/// An integer written with `count` decimal digits. +pub(super) fn digits(count: usize) -> JsonValue { + integer(&"9".repeat(count)) +} + +/// `depth` arrays nested inside one another. +pub(super) fn nested_arrays(depth: usize) -> JsonValue { + (1..depth).fold(JsonValue::Array(Vec::new()), |inner, _| { + JsonValue::Array(vec![inner]) + }) +} diff --git a/crates/sqlbuild-core/src/json/tests/test_dumps.rs b/crates/sqlbuild-core/src/json/tests/test_dumps.rs new file mode 100644 index 0000000000..2d1584eb5d --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/test_dumps.rs @@ -0,0 +1,267 @@ +use crate::json::_helpers::floats::python_repr; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, OrjsonOptions, StdlibJsonOptions}; +use crate::json::tests::helpers::{digits, nested_arrays, non_finite_floats, sample_document}; +use crate::json::tests::test_types::{DumpsTestCase, FloatReprTestCase, IntegerParseTestCase}; + +const BIG: &str = "1180591620717411303424"; +const U64_MAX: &str = "18446744073709551615"; + +#[test] +fn given_python_serializer_options_when_dumping_then_output_matches_python_bytes() { + let stdlib_compact_sorted = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let stdlib_unicode_indented = StdlibJsonOptions { + ensure_ascii: false, + sort_keys: true, + ..StdlibJsonOptions::new(Some(2)) + }; + let test_cases = [ + DumpsTestCase { + description: "json.dumps defaults", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Ok( + r#"{"name": "caf\u00e9 \u2615 \ud83d\ude00", "control": "\u0000\u001f\u007f\t\n\"\\/", "ints": [0, -1, 9223372036854775808, 1180591620717411303424], "floats": [0.1, -0.0, 1e+16, 1e-05, 2.5e-07, 1000000000000000.0, 100.0], "empty": [{}, []], "z": null, "a": true}"#, + ), + }, + DumpsTestCase { + description: "json.dumps indent=2", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + expected_text: Ok( + "{\n \"name\": \"caf\\u00e9 \\u2615 \\ud83d\\ude00\",\n \"control\": \"\\u0000\\u001f\\u007f\\t\\n\\\"\\\\/\",\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 1180591620717411303424\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 1e-05,\n 2.5e-07,\n 1000000000000000.0,\n 100.0\n ],\n \"empty\": [\n {},\n []\n ],\n \"z\": null,\n \"a\": true\n}", + ), + }, + DumpsTestCase { + description: "json.dumps compact sorted ASCII", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(stdlib_compact_sorted), + expected_text: Ok( + r#"{"a":true,"control":"\u0000\u001f\u007f\t\n\"\\/","empty":[{},[]],"floats":[0.1,-0.0,1e+16,1e-05,2.5e-07,1000000000000000.0,100.0],"ints":[0,-1,9223372036854775808,1180591620717411303424],"name":"caf\u00e9 \u2615 \ud83d\ude00","z":null}"#, + ), + }, + DumpsTestCase { + description: "json.dumps ensure_ascii=False indent=2 sorted", + value: sample_document(BIG), + dialect: JsonDialect::Stdlib(stdlib_unicode_indented), + expected_text: Ok( + "{\n \"a\": true,\n \"control\": \"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\n \"empty\": [\n {},\n []\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 1e-05,\n 2.5e-07,\n 1000000000000000.0,\n 100.0\n ],\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 1180591620717411303424\n ],\n \"name\": \"café ☕ 😀\",\n \"z\": null\n}", + ), + }, + DumpsTestCase { + description: "orjson OPT_INDENT_2", + value: sample_document(U64_MAX), + dialect: JsonDialect::Orjson(OrjsonOptions { + indent_2: true, + sort_keys: false, + }), + expected_text: Ok( + "{\n \"name\": \"café ☕ 😀\",\n \"control\": \"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\n \"ints\": [\n 0,\n -1,\n 9223372036854775808,\n 18446744073709551615\n ],\n \"floats\": [\n 0.1,\n -0.0,\n 1e+16,\n 0.00001,\n 2.5e-7,\n 1000000000000000.0,\n 100.0\n ],\n \"empty\": [\n {},\n []\n ],\n \"z\": null,\n \"a\": true\n}", + ), + }, + DumpsTestCase { + description: "orjson OPT_SORT_KEYS", + value: sample_document(U64_MAX), + dialect: JsonDialect::Orjson(OrjsonOptions { + indent_2: false, + sort_keys: true, + }), + expected_text: Ok( + "{\"a\":true,\"control\":\"\\u0000\\u001f\u{7f}\\t\\n\\\"\\\\/\",\"empty\":[{},[]],\"floats\":[0.1,-0.0,1e+16,0.00001,2.5e-7,1000000000000000.0,100.0],\"ints\":[0,-1,9223372036854775808,18446744073709551615],\"name\":\"café ☕ 😀\",\"z\":null}", + ), + }, + DumpsTestCase { + description: "orjson rejects integers beyond 64 bits", + value: sample_document(BIG), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Err(JsonEmitError::IntegerOutOfRange), + }, + DumpsTestCase { + description: "json.dumps writes NaN and infinities", + value: non_finite_floats(), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Ok("[NaN, Infinity, -Infinity]"), + }, + DumpsTestCase { + description: "json.dumps allow_nan=False rejects NaN", + value: non_finite_floats(), + dialect: JsonDialect::Stdlib(StdlibJsonOptions { + allow_nan: false, + ..StdlibJsonOptions::new(None) + }), + expected_text: Err(JsonEmitError::NonFiniteFloat), + }, + DumpsTestCase { + description: "orjson writes non-finite floats as null", + value: non_finite_floats(), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Ok("[null,null,null]"), + }, + DumpsTestCase { + description: "json.dumps refuses integers beyond Python's string conversion limit", + value: digits(4_301), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + expected_text: Err(JsonEmitError::IntegerTooLong), + }, + DumpsTestCase { + description: "deeply nested values are refused instead of exhausting the stack", + value: nested_arrays(1_000), + dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + expected_text: Err(JsonEmitError::NestingTooDeep), + }, + DumpsTestCase { + description: "orjson refuses deeply nested values too", + value: nested_arrays(130), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Err(JsonEmitError::NestingTooDeep), + }, + DumpsTestCase { + description: "nesting within the limit is written", + value: nested_arrays(3), + dialect: JsonDialect::Orjson(OrjsonOptions::default()), + expected_text: Ok("[[[]]]"), + }, + ]; + for test_case in test_cases { + let actual = dumps(&test_case.value, &test_case.dialect); + assert_eq!( + actual.as_deref().map_err(|error| *error), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_floats_when_formatting_python_repr_then_text_matches_cpython() { + let test_cases = [ + FloatReprTestCase { + description: "0.0", + value: 0.0, + expected_repr: "0.0", + }, + FloatReprTestCase { + description: "-0.0", + value: -0.0, + expected_repr: "-0.0", + }, + FloatReprTestCase { + description: "0.0001", + value: 1e-4, + expected_repr: "0.0001", + }, + FloatReprTestCase { + description: "1e-05", + value: 1e-5, + expected_repr: "1e-05", + }, + FloatReprTestCase { + description: "1.5e-300", + value: 1.5e-300, + expected_repr: "1.5e-300", + }, + FloatReprTestCase { + description: "5e-324", + value: 5e-324, + expected_repr: "5e-324", + }, + FloatReprTestCase { + description: "1.2345678901234568e+17", + value: 123_456_789_012_345_680.0, + expected_repr: "1.2345678901234568e+17", + }, + FloatReprTestCase { + description: "9007199254740992.0", + value: 9_007_199_254_740_993.0, + expected_repr: "9007199254740992.0", + }, + FloatReprTestCase { + description: "1.7976931348623157e+308", + value: f64::MAX, + expected_repr: "1.7976931348623157e+308", + }, + FloatReprTestCase { + description: "0.30000000000000004", + value: 0.30000000000000004, + expected_repr: "0.30000000000000004", + }, + FloatReprTestCase { + description: "nan", + value: f64::NAN, + expected_repr: "nan", + }, + FloatReprTestCase { + description: "exact halfway digit rounds half to even", + value: 73_720_622_874_222.62, + expected_repr: "73720622874222.62", + }, + FloatReprTestCase { + description: "-inf", + value: f64::NEG_INFINITY, + expected_repr: "-inf", + }, + ]; + for test_case in test_cases { + assert_eq!( + python_repr(test_case.value), + test_case.expected_repr, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_python_integer_text_when_parsing_then_text_is_canonical() { + let test_cases = [ + IntegerParseTestCase { + description: "leading zeros are dropped", + text: "007", + expected_decimal: Some("7"), + }, + IntegerParseTestCase { + description: "negative zero is zero", + text: "-0", + expected_decimal: Some("0"), + }, + IntegerParseTestCase { + description: "plus sign is dropped", + text: "+42", + expected_decimal: Some("42"), + }, + IntegerParseTestCase { + description: "negative keeps its sign", + text: "-000123", + expected_decimal: Some("-123"), + }, + IntegerParseTestCase { + description: "letters are rejected", + text: "x", + expected_decimal: None, + }, + IntegerParseTestCase { + description: "empty text is rejected", + text: "", + expected_decimal: None, + }, + IntegerParseTestCase { + description: "bare sign is rejected", + text: "-", + expected_decimal: None, + }, + ]; + for test_case in test_cases { + let parsed = JsonInteger::parse(test_case.text); + assert_eq!( + parsed.as_ref().map(JsonInteger::as_str), + test_case.expected_decimal, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/json/tests/test_types.rs b/crates/sqlbuild-core/src/json/tests/test_types.rs new file mode 100644 index 0000000000..2ce319a3b0 --- /dev/null +++ b/crates/sqlbuild-core/src/json/tests/test_types.rs @@ -0,0 +1,21 @@ +use crate::json::errors::JsonEmitError; +use crate::json::models::{JsonDialect, JsonValue}; + +pub(super) struct DumpsTestCase { + pub(super) description: &'static str, + pub(super) value: JsonValue, + pub(super) dialect: JsonDialect, + pub(super) expected_text: Result<&'static str, JsonEmitError>, +} + +pub(super) struct FloatReprTestCase { + pub(super) description: &'static str, + pub(super) value: f64, + pub(super) expected_repr: &'static str, +} + +pub(super) struct IntegerParseTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_decimal: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 3912636b36..d94e5102aa 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -1,6 +1,12 @@ -//! Shared foundations of the SQLBuild native workspace: constants and the panic boundary. +//! Shared foundations of the SQLBuild native workspace: identities, diagnostics, text and JSON. #![forbid(unsafe_code)] pub mod constants; +pub mod json; +pub mod models; pub mod panics; +pub mod text; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/models.rs b/crates/sqlbuild-core/src/models.rs new file mode 100644 index 0000000000..2e5422d295 --- /dev/null +++ b/crates/sqlbuild-core/src/models.rs @@ -0,0 +1,353 @@ +//! Shared compiler identities, diagnostics and typed SQL values mirrored from Python. + +use std::cmp::Ordering; +use std::fmt; + +/// Resource type of a compiled object, Python's `CompiledResourceType`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum CompiledResourceType { + Model, + Source, + Seed, + Udf, + TableFn, + DbtRef, + Audit, + SqlTest, + SqlScenario, +} + +impl CompiledResourceType { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Source => "source", + Self::Seed => "seed", + Self::Udf => "udf", + Self::TableFn => "table_fn", + Self::DbtRef => "dbt_ref", + Self::Audit => "audit", + Self::SqlTest => "sql_test", + Self::SqlScenario => "sql_scenario", + } + } +} + +/// Logical identity of one compiled resource or external dependency, Python's `CompiledObjectKey`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct CompiledObjectKey { + pub resource_type: CompiledResourceType, + pub name: String, +} + +/// Kind of an authored resource in the scope index, Python's `ResourceKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ResourceKind { + Model, + Test, + Scenario, + Function, + Source, + Seed, +} + +impl ResourceKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Test => "test", + Self::Scenario => "scenario", + Self::Function => "function", + Self::Source => "source", + Self::Seed => "seed", + } + } +} + +/// Independent declaration namespace, Python's `DeclarationKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DeclarationKind { + Macro, + Enum, + Constant, + Audit, + SingularAudit, + Schema, + SqlHook, + PythonHook, +} + +impl DeclarationKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Macro => "macro", + Self::Enum => "enum", + Self::Constant => "constant", + Self::Audit => "audit", + Self::SingularAudit => "singular_audit", + Self::Schema => "schema", + Self::SqlHook => "sql_hook", + Self::PythonHook => "python_hook", + } + } +} + +/// Kind-qualified identity of one authored resource, displayed as `kind:name`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ResourceIdentity { + pub kind: ResourceKind, + pub name: String, +} + +/// Declaration identity, displayed as `kind:name` or `kind:owner_kind:owner_name.name`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct DeclarationIdentity { + pub kind: DeclarationKind, + pub name: String, + pub owner: Option, +} + +/// Orders by kind value then name, as Python's `order=True` dataclass compares its fields. +impl Ord for ResourceIdentity { + fn cmp(&self, other: &Self) -> Ordering { + (self.kind.as_str(), self.name.as_str()).cmp(&(other.kind.as_str(), other.name.as_str())) + } +} + +impl PartialOrd for ResourceIdentity { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Orders by kind value, name, then owner; a public declaration sorts before a private twin. +impl Ord for DeclarationIdentity { + fn cmp(&self, other: &Self) -> Ordering { + (self.kind.as_str(), self.name.as_str(), &self.owner).cmp(&( + other.kind.as_str(), + other.name.as_str(), + &other.owner, + )) + } +} + +impl PartialOrd for DeclarationIdentity { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// The text of Python's `format_identity`. +impl fmt::Display for ResourceIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.kind.as_str(), self.name) + } +} + +/// The text of Python's `format_identity`. +impl fmt::Display for DeclarationIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.owner { + None => write!(formatter, "{}:{}", self.kind.as_str(), self.name), + Some(owner) => write!( + formatter, + "{}:{}:{}.{}", + self.kind.as_str(), + owner.kind.as_str(), + owner.name, + self.name + ), + } + } +} + +/// Phase that produced a diagnostic, Python's `DiagnosticPhase`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DiagnosticPhase { + Compile, + Contract, + Rule, + Plan, + Build, + Audit, + Test, + Connection, +} + +impl DiagnosticPhase { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Compile => "compile", + Self::Contract => "contract", + Self::Rule => "rule", + Self::Plan => "plan", + Self::Build => "build", + Self::Audit => "audit", + Self::Test => "test", + Self::Connection => "connection", + } + } +} + +/// Severity of a diagnostic, Python's `DiagnosticSeverity`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DiagnosticSeverity { + Error, + Warning, + Info, +} + +impl DiagnosticSeverity { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::Error => "error", + Self::Warning => "warning", + Self::Info => "info", + } + } +} + +/// An authored location with 1-based lines and code-point columns, Python's `SourceLocation`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SourceLocation { + pub path: String, + pub line: usize, + pub column: usize, + pub end_line: Option, + pub end_column: Option, +} + +/// A secondary location that adds context to a diagnostic, Python's `RelatedLocation`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RelatedLocation { + pub label: String, + pub location: SourceLocation, + pub message: Option, +} + +/// Serial emission position: resource compile order, then stage, then sequence within it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DiagnosticOrderKey { + pub resource_order: u32, + pub stage: u32, + pub sequence: u32, +} + +/// One compile-time diagnostic, Python's `CompilerDiagnostic` plus its serial order key. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Diagnostic { + pub phase: DiagnosticPhase, + pub severity: DiagnosticSeverity, + pub code: String, + pub message: String, + pub resource_type: Option, + pub resource_name: Option, + pub column_name: Option, + pub location: Option, + pub related_locations: Vec, + pub help: Option, + pub notes: Vec, + pub affected_rules: Vec, + pub order_key: DiagnosticOrderKey, +} + +impl Diagnostic { + /// Whether this diagnostic fails the command. + pub fn is_error(&self) -> bool { + self.severity == DiagnosticSeverity::Error + } +} + +/// Closed logical kinds of typed SQL values, Python's `SqlValueKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum SqlValueKind { + String, + Integer, + Boolean, + Float, + Decimal, + Null, + List, + Set, + Object, +} + +impl SqlValueKind { + /// The Python enum value, which is also the serialized form. + pub fn as_str(self) -> &'static str { + match self { + Self::String => "string", + Self::Integer => "integer", + Self::Boolean => "boolean", + Self::Float => "float", + Self::Decimal => "decimal", + Self::Null => "null", + Self::List => "list", + Self::Set => "set", + Self::Object => "object", + } + } +} + +/// Logical type of a typed SQL value; collections carry their element type. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SqlLogicalType { + pub kind: SqlValueKind, + pub element_type: Option>, +} + +impl SqlLogicalType { + /// Python's `display_name`, such as `list`. + pub fn display_name(&self) -> String { + match &self.element_type { + None => self.kind.as_str().to_owned(), + Some(element) => format!("{}<{}>", self.kind.as_str(), element.display_name()), + } + } +} + +/// One validated, canonical typed SQL value, Python's normalized `SqlValue` (see crates/README.md). +#[derive(Clone, Debug, PartialEq)] +pub enum SqlValue { + Null, + Boolean(bool), + Integer(i64), + Float(f64), + Decimal(String), + String(String), + List { + element_type: SqlLogicalType, + values: Vec, + }, + Set { + element_type: SqlLogicalType, + values: Vec, + }, + Object(Vec<(String, SqlValue)>), +} + +impl SqlValue { + /// The logical type of this value. + pub fn logical_type(&self) -> SqlLogicalType { + let (kind, element_type) = match self { + Self::Null => (SqlValueKind::Null, None), + Self::Boolean(_) => (SqlValueKind::Boolean, None), + Self::Integer(_) => (SqlValueKind::Integer, None), + Self::Float(_) => (SqlValueKind::Float, None), + Self::Decimal(_) => (SqlValueKind::Decimal, None), + Self::String(_) => (SqlValueKind::String, None), + Self::List { element_type, .. } => (SqlValueKind::List, Some(element_type)), + Self::Set { element_type, .. } => (SqlValueKind::Set, Some(element_type)), + Self::Object(_) => (SqlValueKind::Object, None), + }; + SqlLogicalType { + kind, + element_type: element_type.map(|element| Box::new(element.clone())), + } + } +} diff --git a/crates/sqlbuild-core/src/tests.rs b/crates/sqlbuild-core/src/tests.rs new file mode 100644 index 0000000000..2ad132af82 --- /dev/null +++ b/crates/sqlbuild-core/src/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_models.rs"] +mod models; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/tests/helpers.rs b/crates/sqlbuild-core/src/tests/helpers.rs new file mode 100644 index 0000000000..a072240428 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/helpers.rs @@ -0,0 +1,64 @@ +use crate::models::{ + DeclarationIdentity, DeclarationKind, Diagnostic, DiagnosticOrderKey, DiagnosticPhase, + DiagnosticSeverity, ResourceIdentity, ResourceKind, SqlLogicalType, SqlValue, SqlValueKind, +}; + +pub(super) fn declaration(kind: DeclarationKind, name: &str) -> DeclarationIdentity { + DeclarationIdentity { + kind, + name: name.to_owned(), + owner: None, + } +} + +pub(super) fn private_constant(owner: &str, name: &str) -> DeclarationIdentity { + DeclarationIdentity { + owner: Some(ResourceIdentity { + kind: ResourceKind::Model, + name: owner.to_owned(), + }), + ..declaration(DeclarationKind::Constant, name) + } +} + +pub(super) fn diagnostic(code: &str, severity: DiagnosticSeverity, order: [u32; 3]) -> Diagnostic { + Diagnostic { + phase: DiagnosticPhase::Compile, + severity, + code: code.to_owned(), + message: format!("{code} message"), + resource_type: None, + resource_name: None, + column_name: None, + location: None, + related_locations: vec![], + help: None, + notes: vec![], + affected_rules: vec![], + order_key: DiagnosticOrderKey { + resource_order: order[0], + stage: order[1], + sequence: order[2], + }, + } +} + +pub(super) fn scalar_type(kind: SqlValueKind) -> SqlLogicalType { + SqlLogicalType { + kind, + element_type: None, + } +} + +pub(super) fn list_of_integer_sets() -> SqlValue { + SqlValue::List { + element_type: SqlLogicalType { + kind: SqlValueKind::Set, + element_type: Some(Box::new(scalar_type(SqlValueKind::Integer))), + }, + values: vec![SqlValue::Set { + element_type: scalar_type(SqlValueKind::Integer), + values: vec![SqlValue::Integer(1), SqlValue::Integer(2)], + }], + } +} diff --git a/crates/sqlbuild-core/src/tests/test_models.rs b/crates/sqlbuild-core/src/tests/test_models.rs new file mode 100644 index 0000000000..f12249ecc2 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/test_models.rs @@ -0,0 +1,112 @@ +use crate::models::{DeclarationKind, Diagnostic, DiagnosticSeverity, SqlValue}; +use crate::tests::helpers::{declaration, diagnostic, list_of_integer_sets, private_constant}; +use crate::tests::test_types::{ + DiagnosticOrderTestCase, IdentityOrderTestCase, IdentityTextTestCase, SqlTypeNameTestCase, +}; + +#[test] +fn given_identities_when_formatting_then_text_matches_python_format_identity() { + let test_cases = [ + IdentityTextTestCase { + description: "public declaration", + identity: declaration(DeclarationKind::SqlHook, "grant_reader"), + expected_text: "sql_hook:grant_reader", + }, + IdentityTextTestCase { + description: "private declaration names its owner", + identity: private_constant("staging.orders", "status"), + expected_text: "constant:model:staging.orders.status", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.identity.to_string(), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_identities_when_sorting_then_order_follows_python_enum_values() { + let test_cases = [IdentityOrderTestCase { + description: "kinds compare by their string values, then names, then owners", + identities: vec![ + declaration(DeclarationKind::Schema, "a"), + private_constant("orders", "b"), + declaration(DeclarationKind::Macro, "a"), + declaration(DeclarationKind::Constant, "b"), + declaration(DeclarationKind::Audit, "z"), + ], + expected_texts: vec![ + "audit:z", + "constant:b", + "constant:model:orders.b", + "macro:a", + "schema:a", + ], + }]; + for test_case in test_cases { + let mut identities = test_case.identities; + identities.sort(); + let texts: Vec = identities.iter().map(ToString::to_string).collect(); + assert_eq!(texts, test_case.expected_texts, "{}", test_case.description); + } +} + +#[test] +fn given_diagnostics_from_parallel_work_when_sorting_by_order_key_then_serial_order_returns() { + let test_cases = [DiagnosticOrderTestCase { + description: "resource order, then stage, then sequence", + diagnostics: vec![ + diagnostic("C003", DiagnosticSeverity::Warning, [1, 0, 0]), + diagnostic("C002", DiagnosticSeverity::Error, [0, 1, 0]), + diagnostic("C001", DiagnosticSeverity::Info, [0, 0, 1]), + diagnostic("C000", DiagnosticSeverity::Error, [0, 0, 0]), + ], + expected_codes: vec!["C000", "C001", "C002", "C003"], + expected_errors: vec![true, false, true, false], + }]; + for test_case in test_cases { + let mut diagnostics = test_case.diagnostics; + diagnostics.sort_by_key(|item| item.order_key); + let codes: Vec<&str> = diagnostics.iter().map(|item| item.code.as_str()).collect(); + let errors: Vec = diagnostics.iter().map(Diagnostic::is_error).collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + assert_eq!( + errors, test_case.expected_errors, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_typed_values_when_naming_types_then_names_match_python_display_name() { + let test_cases = [ + SqlTypeNameTestCase { + description: "nested collections name every element type", + value: list_of_integer_sets(), + expected_name: "list>", + }, + SqlTypeNameTestCase { + description: "scalar", + value: SqlValue::Float(1.5), + expected_name: "float", + }, + SqlTypeNameTestCase { + description: "object has no element type", + value: SqlValue::Object(vec![("status".to_owned(), SqlValue::Null)]), + expected_name: "object", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.value.logical_type().display_name(), + test_case.expected_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/tests/test_types.rs b/crates/sqlbuild-core/src/tests/test_types.rs new file mode 100644 index 0000000000..7550859b12 --- /dev/null +++ b/crates/sqlbuild-core/src/tests/test_types.rs @@ -0,0 +1,26 @@ +use crate::models::{DeclarationIdentity, Diagnostic, SqlValue}; + +pub(super) struct IdentityTextTestCase { + pub(super) description: &'static str, + pub(super) identity: DeclarationIdentity, + pub(super) expected_text: &'static str, +} + +pub(super) struct IdentityOrderTestCase { + pub(super) description: &'static str, + pub(super) identities: Vec, + pub(super) expected_texts: Vec<&'static str>, +} + +pub(super) struct DiagnosticOrderTestCase { + pub(super) description: &'static str, + pub(super) diagnostics: Vec, + pub(super) expected_codes: Vec<&'static str>, + pub(super) expected_errors: Vec, +} + +pub(super) struct SqlTypeNameTestCase { + pub(super) description: &'static str, + pub(super) value: SqlValue, + pub(super) expected_name: &'static str, +} diff --git a/crates/sqlbuild-core/src/text/errors.rs b/crates/sqlbuild-core/src/text/errors.rs new file mode 100644 index 0000000000..d69651f2cf --- /dev/null +++ b/crates/sqlbuild-core/src/text/errors.rs @@ -0,0 +1,7 @@ +//! Errors of decoding authored text. + +/// Authored bytes that are not valid UTF-8; `valid_up_to` is the longest valid prefix. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TextDecodeError { + pub valid_up_to: usize, +} diff --git a/crates/sqlbuild-core/src/text/main/decode_python_text.rs b/crates/sqlbuild-core/src/text/main/decode_python_text.rs new file mode 100644 index 0000000000..386c039cff --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/decode_python_text.rs @@ -0,0 +1,11 @@ +//! Decode authored file bytes the way Python's `Path.read_text(encoding="utf-8")` does. + +use crate::text::errors::TextDecodeError; + +/// Decode strict UTF-8 with universal newlines (`\r\n` and `\r` become `\n`), keeping a BOM. +pub fn decode_python_text(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).map_err(|error| TextDecodeError { + valid_up_to: error.valid_up_to(), + })?; + Ok(text.replace("\r\n", "\n").replace('\r', "\n")) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs new file mode 100644 index 0000000000..57e2d7ac2e --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -0,0 +1 @@ +pub mod decode_python_text; diff --git a/crates/sqlbuild-core/src/text/mod.rs b/crates/sqlbuild-core/src/text/mod.rs new file mode 100644 index 0000000000..ed469c77cd --- /dev/null +++ b/crates/sqlbuild-core/src/text/mod.rs @@ -0,0 +1,6 @@ +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/text/models.rs b/crates/sqlbuild-core/src/text/models.rs new file mode 100644 index 0000000000..7ba97cb723 --- /dev/null +++ b/crates/sqlbuild-core/src/text/models.rs @@ -0,0 +1,101 @@ +//! Positions in authored text, in both UTF-8 bytes and Python code points. + +/// One text position; lines and columns are 1-based and columns count code points. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct TextPosition { + pub byte_offset: usize, + pub char_offset: usize, + pub line: usize, + pub column: usize, +} + +/// A half-open range of text. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TextSpan { + pub start: TextPosition, + pub end: TextPosition, +} + +/// Line starts of one text, converting offsets into Python-compatible lines and columns. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LineIndex<'text> { + text: &'text str, + line_byte_starts: Vec, + line_char_starts: Vec, + char_len: usize, +} + +impl<'text> LineIndex<'text> { + /// Index the line starts of `text`; only `\n` ends a line, as in Python's `str.count`. + pub fn new(text: &'text str) -> Self { + let mut line_byte_starts = vec![0]; + let mut line_char_starts = vec![0]; + let mut char_len = 0; + for (byte_offset, character) in text.char_indices() { + char_len += 1; + if character == '\n' { + line_byte_starts.push(byte_offset + 1); + line_char_starts.push(char_len); + } + } + Self { + text, + line_byte_starts, + line_char_starts, + char_len, + } + } + + /// The position at a UTF-8 byte offset, or `None` past the end or inside a character. + pub fn position_at_byte(&self, byte_offset: usize) -> Option { + if byte_offset > self.text.len() || !self.text.is_char_boundary(byte_offset) { + return None; + } + let line_index = self + .line_byte_starts + .partition_point(|start| *start <= byte_offset) + - 1; + let line_start = self.line_byte_starts[line_index]; + let column = self.text[line_start..byte_offset].chars().count() + 1; + Some(TextPosition { + byte_offset, + char_offset: self.line_char_starts[line_index] + column - 1, + line: line_index + 1, + column, + }) + } + + /// The position at a code-point offset (a Python string index), or `None` past the end. + pub fn position_at_char(&self, char_offset: usize) -> Option { + if char_offset > self.char_len { + return None; + } + let line_index = self + .line_char_starts + .partition_point(|start| *start <= char_offset) + - 1; + let line_start = self.line_byte_starts[line_index]; + let column = char_offset - self.line_char_starts[line_index] + 1; + let byte_offset = self.text[line_start..] + .char_indices() + .nth(column - 1) + .map_or(self.text.len(), |(offset, _)| line_start + offset); + Some(TextPosition { + byte_offset, + char_offset, + line: line_index + 1, + column, + }) + } + + /// The span between two UTF-8 byte offsets, or `None` when either end is not a position. + pub fn span_at_bytes(&self, start: usize, end: usize) -> Option { + if start > end { + return None; + } + Some(TextSpan { + start: self.position_at_byte(start)?, + end: self.position_at_byte(end)?, + }) + } +} diff --git a/crates/sqlbuild-core/src/text/tests.rs b/crates/sqlbuild-core/src/text/tests.rs new file mode 100644 index 0000000000..26f62e1540 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_text.rs"] +mod text; diff --git a/crates/sqlbuild-core/src/text/tests/test_text.rs b/crates/sqlbuild-core/src/text/tests/test_text.rs new file mode 100644 index 0000000000..a59c4164c4 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_text.rs @@ -0,0 +1,135 @@ +use crate::text::main::decode_python_text::decode_python_text; +use crate::text::models::LineIndex; +use crate::text::tests::test_types::{DecodeTestCase, PositionTestCase, SpanTestCase}; + +#[test] +fn given_offsets_when_indexing_text_then_lines_and_columns_match_python() { + let test_cases = [ + PositionTestCase { + description: "start of text", + text: "select 1", + byte_offset: 0, + expected_position: Some((0, 1, 1)), + }, + PositionTestCase { + description: "end of text is addressable", + text: "select 1", + byte_offset: 8, + expected_position: Some((8, 1, 9)), + }, + PositionTestCase { + description: "multi-byte characters count once", + text: "-- café ☕ 😀\nselect é", + byte_offset: 25, + expected_position: Some((19, 2, 8)), + }, + PositionTestCase { + description: "tab counts as one column", + text: "\tselect\t1", + byte_offset: 8, + expected_position: Some((8, 1, 9)), + }, + PositionTestCase { + description: "newline belongs to the line it ends", + text: "a\nb", + byte_offset: 1, + expected_position: Some((1, 1, 2)), + }, + PositionTestCase { + description: "carriage return is an ordinary column", + text: "a\r\nb", + byte_offset: 3, + expected_position: Some((3, 2, 1)), + }, + PositionTestCase { + description: "offset inside a character is rejected", + text: "é", + byte_offset: 1, + expected_position: None, + }, + PositionTestCase { + description: "offset past the end is rejected", + text: "a", + byte_offset: 2, + expected_position: None, + }, + ]; + for test_case in test_cases { + let index = LineIndex::new(test_case.text); + let position = index.position_at_byte(test_case.byte_offset); + let round_trip = position.and_then(|found| index.position_at_char(found.char_offset)); + assert_eq!( + position.map(|found| (found.char_offset, found.line, found.column)), + test_case.expected_position, + "{}", + test_case.description + ); + assert_eq!(round_trip, position, "{} round trip", test_case.description); + } +} + +#[test] +fn given_byte_ranges_when_building_spans_then_both_ends_are_positions() { + let test_cases = [ + SpanTestCase { + description: "range on the second line", + text: "one\ntwo é three", + byte_range: (8, 10), + expected_ends: Some(((2, 5), (2, 6))), + }, + SpanTestCase { + description: "reversed range is rejected", + text: "one", + byte_range: (2, 1), + expected_ends: None, + }, + ]; + for test_case in test_cases { + let span = LineIndex::new(test_case.text) + .span_at_bytes(test_case.byte_range.0, test_case.byte_range.1); + assert_eq!( + span.map(|found| ( + (found.start.line, found.start.column), + (found.end.line, found.end.column) + )), + test_case.expected_ends, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_authored_bytes_when_decoding_then_text_matches_python_read_text() { + let test_cases = [ + DecodeTestCase { + description: "CRLF and lone CR become LF", + bytes: b"a\r\nb\rc\n\r\n", + expected_text: Ok("a\nb\nc\n\n"), + }, + DecodeTestCase { + description: "byte-order mark is kept", + bytes: b"\xef\xbb\xbfselect", + expected_text: Ok("\u{feff}select"), + }, + DecodeTestCase { + description: "encoded surrogate is invalid", + bytes: b"ok\xed\xa0\x80", + expected_text: Err(2), + }, + DecodeTestCase { + description: "truncated sequence is invalid", + bytes: b"caf\xc3", + expected_text: Err(3), + }, + ]; + for test_case in test_cases { + let actual = decode_python_text(test_case.bytes).map_err(|error| error.valid_up_to); + assert_eq!( + actual.as_deref().map_err(|offset| *offset), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs new file mode 100644 index 0000000000..850689e5e1 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -0,0 +1,19 @@ +pub(super) struct PositionTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) byte_offset: usize, + pub(super) expected_position: Option<(usize, usize, usize)>, +} + +pub(super) struct SpanTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) byte_range: (usize, usize), + pub(super) expected_ends: Option<((usize, usize), (usize, usize))>, +} + +pub(super) struct DecodeTestCase { + pub(super) description: &'static str, + pub(super) bytes: &'static [u8], + pub(super) expected_text: Result<&'static str, usize>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 14f33f0798..bf813b9f9f 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -21,6 +21,7 @@ mimalloc.workspace = true pyo3.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-rules.workspace = true sqlbuild-sqltext.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs index 8bd0052338..95f89aec8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs @@ -593,6 +593,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; + crate::bindings::_helpers::oracles::register(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 83143c2dbf..d1e72c2cfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,5 +3,6 @@ mod catalog; pub(crate) mod conversions; pub(crate) mod functions; mod normalization_results; +mod oracles; pub(crate) mod panics; mod positions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs new file mode 100644 index 0000000000..1c422613db --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/oracles.rs @@ -0,0 +1,236 @@ +//! Internal hooks that let Python tests compare native foundations with Python byte for byte. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use serde_json::{Value, json}; +use sqlbuild_config::errors::ConfigError; +use sqlbuild_config::models::{ConfigDate, ConfigTime, ConfigValue}; +use sqlbuild_config::project::main::read_local_config::read_local_config; +use sqlbuild_config::project::main::read_project_config::read_project_config; +use sqlbuild_config::toml::main::load_toml::load_toml; +use sqlbuild_config::yaml::main::safe_load::safe_load; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{ + JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, +}; +use sqlbuild_core::text::main::decode_python_text::decode_python_text; +use sqlbuild_core::text::models::LineIndex; +use std::path::Path; + +fn oracle_error(message: impl std::fmt::Display) -> PyErr { + PyValueError::new_err(message.to_string()) +} + +fn text_field<'value>(value: &'value Value, key: &str) -> PyResult<&'value str> { + value[key] + .as_str() + .ok_or_else(|| oracle_error(format!("expected string field {key}"))) +} + +fn flag(value: &Value, key: &str) -> PyResult { + value[key] + .as_bool() + .ok_or_else(|| oracle_error(format!("expected boolean field {key}"))) +} + +fn dialect(spec: &Value) -> PyResult { + if let Some(options) = spec.get("orjson") { + return Ok(JsonDialect::Orjson(OrjsonOptions { + indent_2: flag(options, "indent_2")?, + sort_keys: flag(options, "sort_keys")?, + })); + } + Ok(JsonDialect::Stdlib(StdlibJsonOptions { + indent: spec["indent"].as_str().map(str::to_owned), + item_separator: text_field(&spec["separators"], "item")?.to_owned(), + key_separator: text_field(&spec["separators"], "key")?.to_owned(), + ensure_ascii: flag(spec, "ensure_ascii")?, + sort_keys: flag(spec, "sort_keys")?, + allow_nan: flag(spec, "allow_nan")?, + })) +} + +fn float_from_bits(bits: &Value) -> PyResult { + let text = bits + .as_str() + .ok_or_else(|| oracle_error("expected float bits"))?; + text.parse::() + .map(f64::from_bits) + .map_err(oracle_error) +} + +/// Decode `[tag, payload]`, the tagged form the Python oracle tests write. +fn tagged_value(tagged: &Value) -> PyResult { + let payload = &tagged[1]; + match tagged[0].as_str() { + Some("null") => Ok(JsonValue::Null), + Some("bool") => payload + .as_bool() + .map(JsonValue::Bool) + .ok_or_else(|| oracle_error("expected boolean")), + Some("int") => payload + .as_str() + .and_then(JsonInteger::parse) + .map(JsonValue::Integer) + .ok_or_else(|| oracle_error("expected decimal integer")), + Some("float") => float_from_bits(payload).map(JsonValue::Float), + Some("str") => payload + .as_str() + .map(|text| JsonValue::String(text.to_owned())) + .ok_or_else(|| oracle_error("expected string")), + Some("list") => tagged_items(payload)? + .iter() + .map(tagged_value) + .collect::>>() + .map(JsonValue::Array), + Some("dict") => tagged_items(payload)? + .iter() + .map(|entry| { + let key = entry[0] + .as_str() + .ok_or_else(|| oracle_error("expected string key"))?; + Ok((key.to_owned(), tagged_value(&entry[1])?)) + }) + .collect::>>() + .map(JsonValue::Object), + _ => Err(oracle_error("unknown tagged value")), + } +} + +fn tagged_items(payload: &Value) -> PyResult<&Vec> { + payload + .as_array() + .ok_or_else(|| oracle_error("expected tagged items")) +} + +/// Serialize a tagged value natively, or return `error:` for a refused value. +#[pyfunction] +fn _oracle_json_dumps(dialect_json: &str, value_json: &str) -> PyResult { + let spec: Value = serde_json::from_str(dialect_json).map_err(oracle_error)?; + let tagged: Value = serde_json::from_str(value_json).map_err(oracle_error)?; + Ok(dumps(&tagged_value(&tagged)?, &dialect(&spec)?) + .unwrap_or_else(|error| format!("error:{error:?}"))) +} + +type OraclePosition = Option<(usize, usize, usize)>; + +fn oracle_position(index: &LineIndex<'_>, byte_offset: usize) -> OraclePosition { + let position = index.position_at_byte(byte_offset)?; + Some((position.char_offset, position.line, position.column)) +} + +/// Decode authored bytes and report `(char_offset, line, column)` for each byte offset. +#[pyfunction] +fn _oracle_text_positions( + data: &[u8], + byte_offsets: Vec, +) -> PyResult<(String, Vec)> { + let text = decode_python_text(data).map_err(|error| oracle_error(error.valid_up_to))?; + let positions = { + let index = LineIndex::new(&text); + byte_offsets + .into_iter() + .map(|offset| oracle_position(&index, offset)) + .collect() + }; + Ok((text, positions)) +} + +fn date_parts(date: ConfigDate) -> Value { + json!([date.year, date.month, date.day]) +} + +fn time_parts(time: ConfigTime) -> Value { + json!([time.hour, time.minute, time.second, time.microsecond]) +} + +fn float_canonical(value: f64) -> Value { + if value.is_nan() { + return json!(["float", "nan"]); + } + json!(["float", value.to_bits().to_string()]) +} + +/// The tagged canonical form the Python oracle tests build from `tomllib` and PyYAML values. +fn canonical(value: &ConfigValue) -> Value { + match value { + ConfigValue::Null => json!(["null"]), + ConfigValue::Bool(flag) => json!(["bool", flag]), + ConfigValue::Integer(number) => json!(["int", number.to_string()]), + ConfigValue::BigInteger(text) => json!(["int", text]), + ConfigValue::Float(number) => float_canonical(*number), + ConfigValue::String(text) => json!(["str", text]), + ConfigValue::Date(date) => json!(["date", date_parts(*date)]), + ConfigValue::DateTime(moment) => json!([ + "datetime", + date_parts(moment.date), + time_parts(moment.time), + moment.utc_offset_seconds + ]), + ConfigValue::Time(time) => json!(["time", time_parts(*time)]), + ConfigValue::List(items) => { + json!(["list", items.iter().map(canonical).collect::>()]) + } + ConfigValue::Map(entries) => json!([ + "map", + entries + .iter() + .map(|(key, item)| json!([canonical(key), canonical(item)])) + .collect::>() + ]), + } +} + +fn outcome(result: Result) -> String { + result + .unwrap_or_else(|error| json!({"error": format!("{:?}", error.kind)})) + .to_string() +} + +/// Load YAML natively and return its canonical form, or `{"error": kind}`. +#[pyfunction] +fn _oracle_yaml_load(text: &str) -> String { + outcome(safe_load(text).map(|value| canonical(&value))) +} + +/// Load TOML natively and return its canonical form, or `{"error": kind}`. +#[pyfunction] +fn _oracle_toml_load(text: &str) -> String { + outcome(load_toml(text).map(|value| canonical(&value))) +} + +/// Read a project's discovery configuration natively, or return `{"error": kind}`. +#[pyfunction] +fn _oracle_project_config(project_dir: &str) -> String { + let directory = Path::new(project_dir); + outcome(read_project_config(directory).and_then(|project| { + let local = read_local_config(directory)?; + Ok(json!({ + "name": project.name, + "adapter": project.adapter, + "default_target": project.default_target, + "sql_analysis": project.settings.sql_analysis, + "require_sql_analysis": project.settings.require_sql_analysis, + "enforce_placement": project.enforce_placement, + "enforce_explicit_references": project.enforce_explicit_references, + "vars": project.vars, + "path_default_keys": project.path_defaults.iter().map(|(key, _)| key).collect::>(), + "target_names": project.target_names, + "local_target": local.target, + "local_adapter": local.adapter, + "local_sql_analysis": local.sql_analysis, + "local_vars": local.vars, + "local_target_names": local.target_names, + })) + })) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(_oracle_json_dumps, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_yaml_load, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_toml_load, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_project_config, module)?)?; + Ok(()) +} diff --git a/fensu.toml b/fensu.toml index 89e4d7f529..de66a371f9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -305,6 +305,7 @@ remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ "sqlbuild-analysis", + "sqlbuild-config", "sqlbuild-core", "sqlbuild-python", "sqlbuild-rules", diff --git a/pyproject.toml b/pyproject.toml index 1b03327fb7..2cee655b4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.12" dependencies = [ "duckdb>=1.3.2", "filelock>=3.20.3", - "orjson>=3.10.0", + "orjson>=3.11.7", "polyglot-sql-chio==0.13.8", "pydantic-settings>=2,<3", "PyYAML>=6.0.2", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3eb370a343..532bca8ae0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -49,3 +49,12 @@ def substitute_static_project_vars( def extract_static_sql_references( sql: str, ) -> list[tuple[str, str, str | None, int | None]] | None: ... + +# Internal oracle hooks for tests that compare native foundations with Python; not an API. +def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... +def _oracle_text_positions( + data: bytes, byte_offsets: list[int] +) -> tuple[str, list[tuple[int, int, int] | None]]: ... +def _oracle_yaml_load(text: str) -> str: ... +def _oracle_toml_load(text: str) -> str: ... +def _oracle_project_config(project_dir: str) -> str: ... diff --git a/tests/integration/src/sqlbuild/compiler/_test_types.py b/tests/integration/src/sqlbuild/compiler/_test_types.py new file mode 100644 index 0000000000..f17956cecf --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/_test_types.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JsonOracleTestCase: + """One Python serializer configuration compared with the native emitter on seeded values.""" + + description: str + serializer: str + seed: int + case_count: int + indent: int | str | None = None + separators: tuple[str, str] | None = None + ensure_ascii: bool = True + sort_keys: bool = False + allow_nan: bool = True + orjson_option: int = 0 + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class TextOracleTestCase: + """Seeded authored files whose native decoding and positions are compared with Python.""" + + description: str + seed: int + case_count: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class OrjsonFloatLayoutTestCase: + """A float whose orjson text depends on the orjson version the native emitter reproduces.""" + + description: str + value: float + expected_text: str diff --git a/tests/integration/src/sqlbuild/compiler/discovery/__init__.py b/tests/integration/src/sqlbuild/compiler/discovery/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py new file mode 100644 index 0000000000..445a81334c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedDocumentOracleTestCase: + """Seeded generated documents loaded natively and by the Python library they mirror.""" + + description: str + generator: str + seed: int + case_count: int + expected_maximum_deferred: int + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class RepositoryFileOracleTestCase: + """Repository files of one kind loaded natively and by the Python loader.""" + + description: str + pattern: str + expected_minimum_files: int + expected_deferred: int = 0 + expected_mismatches: tuple[tuple[object, object, object], ...] = () + + +@dataclass(frozen=True) +class HostileDocumentOracleTestCase: + """A document built to exhaust a naive reader, which the native reader must defer quickly.""" + + description: str + document: str + expected_deferred: bool = True + expected_maximum_seconds: float = 1.0 + + +@dataclass(frozen=True) +class LargeDocumentOracleTestCase: + """A large single-line document that must load in linear time with PyYAML's value.""" + + description: str + document: str + expected_maximum_seconds: float = 2.0 diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py new file mode 100644 index 0000000000..86242681ed --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -0,0 +1,581 @@ +"""Seeded YAML and TOML generators and canonical values for the native configuration oracles.""" + +from __future__ import annotations + +import datetime +import json +import math +import random +import struct +import tomllib +from collections.abc import Callable +from contextlib import suppress +from itertools import compress +from pathlib import Path +from typing import Any + +import yaml + +from sqlbuild import _native +from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text + +REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] +EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".venv", "node_modules", "target", ".git"}) +PYTHON_ERROR: dict[str, str] = {"error": "rejected"} +DEFERRED: dict[str, str] = {"error": "deferred to Python"} +LOADERS_DISAGREE: dict[str, str] = {"error": "PyYAML and LibYAML disagree"} +UNCHECKED_PYTHON_OUTCOMES: tuple[dict[str, str], ...] = (PYTHON_ERROR, LOADERS_DISAGREE) +YAML_LINE_BREAKS: dict[int, None] = {0x85: None, 0x2028: None, 0x2029: None} +SCALAR_FRAGMENTS: tuple[str, ...] = ( + "0", "1", "7", "8", "9", "00", "12", "59", "60", "_", ".", ":", "e", "E", "e+", "E-", + "+", "-", "x", "b", "o", "0x", "0b", "0o", "a", "f", "F", "inf", "Inf", ".inf", ".nan", + "NaN", "yes", "Yes", "NO", "on", "ON", "off", "true", "True", "FALSE", "null", "Null", + "NULL", "~", "<<", "=", "y", "n", "T", "t", " ", "Z", "2001", "-12", "-2", "-14", "T21", + ":59", ":43", ".10", "-05:00", "+5", " -5", "1_0", "é", "#", "'", '"', +) # fmt: skip +KNOWN_SCALARS: tuple[str, ...] = ( + "yes", "No", "oN", "~", "", "<<", "=", "017", "0o17", "08", "0b101", "0x1F", "0x_", + "1_000", "190:20:30", "1:60", "-190:20:30.5", "1e5", "1.0e5", "1.0e+5", ".5", "5.", "+.5", + "-0.0", ".NaN", "-.inf", "2001-12-14", "2001-1-4", "2002-02-30", "2001-12-14t21:59:43.10-05:00", + "2001-12-14 21:59:43.10 -5", "2001-12-14 21:59:43.1234567Z", "2001-12-14T24:00:00", + "2001-12-14 1:02:03 +23:59", "99999999999999999999999", "-0b_1", "0_", "_1", "1__2.3_", +) # fmt: skip +SCALAR_CONTEXTS: tuple[str, ...] = ( + "key: {plain}", + "- {plain}", + "[{plain}, 1]", + "{plain}: value", + "key: '{single}'", + 'key: "{double}"', + "key: ! {plain}", + "key: ! '{single}'", + "key: !!int {plain}", + "key: !!float {plain}", + "key: !!bool {plain}", + "key: !!str {plain}", + "key: !!null {plain}", + "key: !!timestamp {plain}", + "? {plain}\n: 1", +) +DOCUMENT_TEMPLATES: tuple[str, ...] = ( + "base{n}: &anchor{n}\n shared: {a}\n other: {b}\nitem{n}:\n <<: *anchor{n}\n other: {c}\n", + "list{n}:\n - &item{n} {a}\n - *item{n}\n - [{b}, {c}]\n", + "merged{n}:\n <<: [{{x: {a}}}, {{x: {b}, y: {c}}}]\n z: {a}\n", + "flow{n}: {{k{n}: {a}, j{n}: [{b}, {{m: {c}}}]}}\n", + "literal{n}: |\n {a}\n {b}\n\n {c}\nfolded{n}: >-\n {a}\n {b}\n", + "? complex{n}\n: {a}\n# comment {b}\nafter{n}: {c} # trailing\n", + "{a}: first\n{a}: second\n", + "keep{n}: |+\n {a}\n\n {b}\n\n", + "strip{n}: >-\n {a}\n\n {b}\n \n", + "seq{n}:\n- |\n {a}\n- >+\n {b}\n \n", + "nested{n}:\n inner: |\n {a}\n {b}\n \n", + "indented{n}:\n - |2-\n {a}\n {c}\n", + "empty{n}: |\n \n\nafter{n}: {b}\n", +) +DOCUMENT_TAILS: tuple[str, ...] = ( + "", "\n", "\n\n", " ", " ", "\n ", "\n ", "\n \n", " \n ", "\n# end", "\n...\n", +) # fmt: skip +BYTE_ORDER_MARKS: tuple[str, ...] = ("",) * 19 + ("\ufeff",) + + +def yaml_text(*, rng: random.Random, max_length: int) -> str: + """Return random text without the line breaks only YAML 1.1 recognizes.""" + + return random_text(rng=rng, max_length=max_length).translate(YAML_LINE_BREAKS) + + +def random_scalar(*, rng: random.Random) -> str: + """Return a near-miss plain scalar built from resolver-sensitive fragments.""" + + pieces: list[str] = [rng.choice(SCALAR_FRAGMENTS) for _ in range(rng.randint(1, 4))] + return rng.choice(("".join(pieces), rng.choice(KNOWN_SCALARS))) + + +def scalar_document(*, rng: random.Random) -> str: + """Return a scalar embedded in a plain, quoted or tagged YAML context.""" + + scalar: str = random_scalar(rng=rng) + return rng.choice(SCALAR_CONTEXTS).format( + plain=scalar, single=scalar.replace("'", "''"), double=scalar.replace('"', "'") + ) + + +def anchored_document(*, rng: random.Random) -> str: + """Return a block document with anchors, aliases, merge keys, literals and comments.""" + + return "".join( + rng.choice(DOCUMENT_TEMPLATES).format( + n=index, a=random_scalar(rng=rng), b=random_scalar(rng=rng), c=random_scalar(rng=rng) + ) + for index in range(rng.randint(1, 4)) + ) + + +def _yaml_key(rng: random.Random) -> object: + return rng.choice( + ( + yaml_text(rng=rng, max_length=6), + rng.randint(-5, 5), + rng.random() < 0.5, + None, + datetime.date(2000 + rng.randint(0, 30), rng.randint(1, 12), rng.randint(1, 28)), + yaml_text(rng=rng, max_length=12), + ) + ) + + +def _yaml_datetime(rng: random.Random) -> object: + offset: datetime.timedelta = datetime.timedelta(minutes=rng.randint(-1439, 1439)) + zone: datetime.tzinfo | None = rng.choice((None, datetime.timezone(offset), datetime.UTC)) + return datetime.datetime( + rng.randint(1, 9999), + rng.randint(1, 12), + rng.randint(1, 28), + rng.randint(0, 23), + rng.randint(0, 59), + rng.randint(0, 59), + rng.choice((0, rng.randint(0, 999_999))), + tzinfo=zone, + ) + + +YAML_SCALARS: tuple[Callable[[random.Random], object], ...] = ( + lambda rng: None, + lambda rng: rng.random() < 0.5, + lambda rng: rng.randint(-(2**70), 2**70), + lambda rng: random_float(rng=rng), + lambda rng: yaml_text(rng=rng, max_length=30), + lambda rng: random_scalar(rng=rng), + lambda rng: datetime.date(rng.randint(1, 9999), rng.randint(1, 12), rng.randint(1, 28)), + _yaml_datetime, +) + + +def _yaml_list(rng: random.Random, depth: int) -> object: + return [random_yaml_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 4))] + + +def _yaml_dict(rng: random.Random, depth: int) -> object: + return { + _yaml_key(rng): random_yaml_value(rng=rng, depth=depth - 1) + for _ in range(rng.randint(0, 4)) + } + + +YAML_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + *(lambda rng, depth, scalar=scalar: scalar(rng) for scalar in YAML_SCALARS), + _yaml_list, + _yaml_dict, + _yaml_list, + _yaml_dict, +) + + +def random_yaml_value(*, rng: random.Random, depth: int) -> object: + """Return a random value of the types PyYAML's safe dumper writes.""" + + available: int = len(YAML_SCALARS) + 4 * min(depth, 1) + return rng.choice(YAML_GENERATORS[:available])(rng, depth) + + +def dumped_document(*, rng: random.Random) -> str: + """Return a document written by PyYAML's safe dumper in a random style.""" + + return yaml.safe_dump( + random_yaml_value(rng=rng, depth=3), + default_flow_style=rng.choice((None, True, False)), + default_style=rng.choice((None, None, '"', "'")), + width=rng.choice((12, 80, 4096)), + allow_unicode=rng.random() < 0.5, + explicit_start=rng.random() < 0.3, + sort_keys=False, + ) + + +def _alias_chain(links: int) -> str: + return "a0: &a0 [x]\n" + "".join( + f"a{link}: &a{link} [*a{link - 1}]\n" for link in range(1, links) + ) + + +def _billion_laughs(levels: int) -> str: + return 'a0: &a0 "lol"\n' + "".join( + f"a{level}: &a{level} [{', '.join([f'*a{level - 1}'] * 10)}]\n" + for level in range(1, levels) + ) + + +def _merge_chain(levels: int) -> str: + return "m0: &m0 {k0: 1}\n" + "".join( + f"m{level}: &m{level} {{<<: [*m{level - 1}, *m{level - 1}], k{level}: 1}}\n" + for level in range(1, levels) + ) + + +HOSTILE_YAML_DOCUMENTS: dict[str, Callable[[], str]] = { + "alias chain": lambda: _alias_chain(10_000), + "billion laughs": lambda: _billion_laughs(12), + "merge chain": lambda: _merge_chain(30), + "deep flow": lambda: "a: " + "[" * 100_000 + "]" * 100_000, + "deep block": lambda: "- " * 100_000, + "long integer": lambda: "a: " + "9" * 4_301, +} +LARGE_YAML_DOCUMENTS: dict[str, Callable[[], str]] = { + "flow mapping": lambda: "{" + ", ".join(f"k{index}: {index}" for index in range(100_000)) + "}", + "flow sequence": lambda: "[" + ",".join("1" for _ in range(200_000)) + "]", + "anchored flow sequence": lambda: ( + "[" + ", ".join(f"&a{index} x{index}" for index in range(50_000)) + "]" + ), +} +HOSTILE_TOML_DOCUMENTS: dict[str, Callable[[], str]] = { + "deep arrays": lambda: "a = " + "[" * 10_000 + "]" * 10_000, + "deep inline tables": lambda: "a = " + "{b = " * 10_000 + "}" * 10_000, + "long dotted key": lambda: "a" + ".a" * 100_000 + " = 1", + "long table header": lambda: "[a" + ".a" * 100_000 + "]", +} + + +def finished_document(*, rng: random.Random, text: str) -> str: + """Return `text` with or without its final line break, trailing blank lines or a BOM.""" + + trimmed: str = text.rstrip("\n") + ending: str = rng.choice((text, trimmed, trimmed + rng.choice(DOCUMENT_TAILS))) + return rng.choice(BYTE_ORDER_MARKS) + ending + + +YAML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "scalars": lambda rng: finished_document(rng=rng, text=scalar_document(rng=rng)), + "anchors": lambda rng: finished_document(rng=rng, text=anchored_document(rng=rng)), + "dumped": lambda rng: finished_document(rng=rng, text=dumped_document(rng=rng)), +} + + +def _toml_key(rng: random.Random, index: int) -> str: + bare: str = f"k{index}_{rng.choice(('a', 'B', '1', '-', '_'))}" + quoted: str = json.dumps(yaml_text(rng=rng, max_length=5) + str(index), ensure_ascii=False) + literal: str = f"'{rng.choice(('é', 'x y', 'dot.ted'))}{index}'" + return rng.choice((bare, bare, quoted, literal)) + + +def _toml_datetime(rng: random.Random) -> str: + date: str = f"{rng.randint(1, 9999):04d}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" + time: str = f"{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + fraction: str = rng.choice(("", f".{rng.randint(0, 10**9)}", ".5")) + offset: str = rng.choice( + ("", "Z", "z", f"{rng.choice('+-')}{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}") + ) + return rng.choice( + (date, f"{time}{fraction}", f"{date}T{time}{fraction}{offset}", f"{date} {time}{offset}") + ) + + +def _toml_float(rng: random.Random) -> str: + value: float = random_float(rng=rng) + exponent: str = f"{rng.randint(-99, 99)}.{rng.randint(0, 99)}e{rng.randint(-30, 30)}" + return rng.choice((repr(value), exponent, "1_000.5", "-0.0", "+inf", "-nan")) + + +TOML_ESCAPES: tuple[str, ...] = ( + "\\b", "\\t", "\\n", "\\f", "\\r", '\\"', "\\\\", "\\u00e9", "\\U0001F600", +) # fmt: skip +TOML_STRING_LITERALS: tuple[str, ...] = ( + "'C:\\path'", + "'a \"q\"'", + "'é'", + '"""\nline"""', + '"""\none\\\n two"""', + "'''\nraw \\n'''", + "'''\ntwo\nlines'''", +) +TOML_1_1_SYNTAX: tuple[str, ...] = ( + '"\\e"', + '"\\x41"', + "{x = 1,}", + "{\n x = 1\n}", + "{x = 1, # note\n}", + "07:32", + "1979-05-27T07:32Z", + "1979-05-27 07:32", +) +TOML_TABLE_NAMES: tuple[str, ...] = ("p", "p.q", "p.q.r", "p.s", "t", "t.u", '"p".v') +TOML_SCALARS: tuple[Callable[[random.Random], str], ...] = ( + lambda rng: rng.choice((*TOML_1_1_SYNTAX, str(2**64), str(-(2**80)))), + lambda rng: str(rng.randint(-(2**63), 2**63 - 1)), + lambda rng: rng.choice(("0x1F", "0o17", "0b101", "1_000", "+7", "-0")), + _toml_float, + lambda rng: rng.choice(("true", "false")), + lambda rng: json.dumps(yaml_text(rng=rng, max_length=10), ensure_ascii=False), + lambda rng: '"' + "".join(rng.choice(TOML_ESCAPES) for _ in range(rng.randint(0, 3))) + 'x"', + lambda rng: rng.choice(TOML_STRING_LITERALS), + _toml_datetime, +) + + +def _toml_inline(rng: random.Random, depth: int) -> str: + entries: str = ", ".join( + f"{_toml_key(rng, index)} = {random_toml_value(rng=rng, depth=depth - 1)}" + for index in range(rng.randint(0, 3)) + ) + return "{" + entries + "}" + + +def _toml_array(rng: random.Random, depth: int) -> str: + values: list[str] = [ + random_toml_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 3)) + ] + return "[" + rng.choice((", ", ",\n ")).join(values) + rng.choice(("", ",")) + "]" + + +TOML_GENERATORS: tuple[Callable[[random.Random, int], str], ...] = ( + *(lambda rng, depth, scalar=scalar: scalar(rng) for scalar in TOML_SCALARS), + _toml_inline, + _toml_array, +) + + +def random_toml_value(*, rng: random.Random, depth: int) -> str: + """Return the TOML text of a random value.""" + + available: int = len(TOML_SCALARS) + 2 * min(depth, 1) + return rng.choice(TOML_GENERATORS[:available])(rng, depth) + + +def _toml_pairs(rng: random.Random, prefix: str) -> str: + return "".join( + f"{prefix}{_toml_key(rng, index)} = {random_toml_value(rng=rng, depth=2)}" + f"{rng.choice(('', ' # note'))}\n" + for index in range(rng.randint(0, 4)) + ) + + +def toml_document(*, rng: random.Random) -> str: + """Return a TOML document with root keys, dotted keys, tables and arrays of tables. + + Table names repeat across sections, so implicit tables are later declared explicitly or + extended through dotted keys, as `tomllib` orders them by first creation. + """ + + sections: list[str] = [ + rng.choice( + ( + f"[table{index}]\n{_toml_pairs(rng, '')}", + f"[parent.child{index}]\n{_toml_pairs(rng, '')}", + f"[[rows]]\n{_toml_pairs(rng, '')}", + f"{_toml_pairs(rng, f'dotted{index}.')}", + f"[{rng.choice(TOML_TABLE_NAMES)}]\n{_toml_pairs(rng, '')}", + f"[[{rng.choice(TOML_TABLE_NAMES)}]]\n{_toml_pairs(rng, '')}", + f"{_toml_pairs(rng, rng.choice(TOML_TABLE_NAMES) + '.')}", + f"[late{index}.inner]\n{_toml_pairs(rng, '')}[side{index}]\n" + f"{_toml_pairs(rng, '')}[late{index}]\n{_toml_pairs(rng, '')}", + f"[[late{index}.rows]]\n{_toml_pairs(rng, '')}[late{index}]\n" + f"{_toml_pairs(rng, f'more{index}.')}", + ) + ) + for index in range(rng.randint(0, 6)) + ] + document: str = _toml_pairs(rng, "") + "\n".join(sections) + line_endings: str = rng.choice((document, document, document.replace("\n", "\r\n"))) + return rng.choice(BYTE_ORDER_MARKS) + line_endings + + +TOML_NAMESPACE_NAMES: tuple[str, ...] = ( + "a", "b", "a.b", "a.c", "b.a", "a.b.c", "a.b.c.d", "b.c.d", '"a".b', "a.'b'.c", +) # fmt: skip +TOML_NAMESPACE_VALUES: tuple[str, ...] = ( + "1", "{}", "{x = 1}", "{c = {d = 1}}", "[]", "[{x = 1}]", "{b.c = 1}", "[1, 2]", +) # fmt: skip +TOML_NAMESPACE_STATEMENTS: tuple[Callable[[random.Random], str], ...] = ( + lambda rng: f"[{rng.choice(TOML_NAMESPACE_NAMES)}]", + lambda rng: f"[[{rng.choice(TOML_NAMESPACE_NAMES)}]]", + lambda rng: f"{rng.choice(TOML_NAMESPACE_NAMES)} = {rng.choice(TOML_NAMESPACE_VALUES)}", + lambda rng: f"{rng.choice(TOML_NAMESPACE_NAMES)}.k{rng.randint(0, 2)} = 1", + lambda rng: ( + f"x = {{{rng.choice(TOML_NAMESPACE_NAMES)} = 1, {rng.choice(TOML_NAMESPACE_NAMES)} = 2}}" + ), +) + + +def toml_namespace_document(*, rng: random.Random) -> str: + """Return statements that reopen tables, arrays of tables and dotted keys in random orders.""" + + return "\n".join(rng.choice(TOML_NAMESPACE_STATEMENTS)(rng) for _ in range(rng.randint(1, 8))) + + +TOML_DOCUMENT_GENERATORS: dict[str, Callable[[random.Random], str]] = { + "toml": lambda rng: toml_document(rng=rng), + "toml namespaces": lambda rng: toml_namespace_document(rng=rng), +} + + +def _float_canonical(value: float) -> list[object]: + bits: str = str(struct.unpack(" int | None: + offset: datetime.timedelta | None = value.utcoffset() + return OFFSET_SECONDS[type(offset)](offset) + + +CANONICAL: dict[type, Callable[[Any], object]] = { + type(None): lambda value: ["null"], + bool: lambda value: ["bool", value], + int: lambda value: ["int", str(value)], + float: _float_canonical, + str: lambda value: ["str", value], + datetime.date: lambda value: ["date", [value.year, value.month, value.day]], + datetime.datetime: lambda value: [ + "datetime", + [value.year, value.month, value.day], + [value.hour, value.minute, value.second, value.microsecond], + _utc_offset(value), + ], + datetime.time: lambda value: [ + "time", + [value.hour, value.minute, value.second, value.microsecond], + ], + list: lambda value: ["list", [canonical(item) for item in value]], + dict: lambda value: ["map", [[canonical(key), canonical(item)] for key, item in value.items()]], +} + + +def canonical(value: object) -> object: + """Return the tagged canonical form the native oracle hooks also produce.""" + + return CANONICAL[type(value)](value) + + +def python_yaml_outcome(*, text: str, loader: type[yaml.SafeLoader]) -> object: + """Return PyYAML's canonical value, or the rejection marker when it raises.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress( + yaml.YAMLError, ValueError, TypeError, OverflowError, AttributeError, KeyError, IndexError + ): + outcome[0] = canonical(yaml.load(text, Loader=loader)) + return outcome[0] + + +def expected_yaml_outcome(*, text: str) -> object: + """Return the value both PyYAML loaders agree on, or the marker that native must defer.""" + + pure: object = python_yaml_outcome(text=text, loader=yaml.SafeLoader) + libyaml: object = python_yaml_outcome(text=text, loader=yaml.CSafeLoader) + return {True: pure, False: LOADERS_DISAGREE}[pure == libyaml] + + +def python_toml_outcome(*, text: str) -> object: + """Return `tomllib`'s canonical value, or the rejection marker when it raises.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress(tomllib.TOMLDecodeError, ValueError): + outcome[0] = canonical(tomllib.loads(text)) + return outcome[0] + + +def native_outcome(payload: str) -> object: + """Return the native canonical value, or the marker of its failure kind.""" + + parsed: object = json.loads(payload) + return {list: parsed, dict: DEFERRED}[type(parsed)] + + +def native_yaml_outcome(*, text: str) -> object: + """Return the native YAML loader's outcome for `text`.""" + + return native_outcome(_native._oracle_yaml_load(text)) + + +def native_toml_outcome(*, text: str) -> object: + """Return the native TOML loader's outcome for `text`.""" + + return native_outcome(_native._oracle_toml_load(text)) + + +def repository_files(*, pattern: str) -> list[Path]: + """Return tracked-looking repository files matching `pattern`, outside build directories.""" + + return sorted( + filter( + lambda path: EXCLUDED_DIRECTORIES.isdisjoint(path.relative_to(REPOSITORY_ROOT).parts), + REPOSITORY_ROOT.glob(pattern), + ) + ) + + +def _python_project_fields(project: ProjectConfig, local: LocalConfig) -> dict[str, object]: + return { + "name": project.name, + "adapter": project.adapter, + "default_target": project.default_target, + "sql_analysis": project.settings.sql_analysis, + "require_sql_analysis": project.settings.require_sql_analysis, + "enforce_placement": project.scopes.enforce_placement, + "enforce_explicit_references": project.references.enforce_explicit, + "vars": [list(item) for item in project.vars.items()], + "path_default_keys": list(project.path_defaults), + "target_names": list(project.targets), + "local_target": local.target, + "local_adapter": local.adapter, + "local_sql_analysis": {True: local.settings.sql_analysis, False: None}[ + "sql_analysis" in local.setting_overrides + ], + "local_vars": [list(item) for item in local.vars.items()], + "local_target_names": list(local.targets), + } + + +def python_project_outcome(*, project_dir: Path) -> object: + """Return the discovery fields Python's loaders read, or the rejection marker.""" + + outcome: list[object] = [PYTHON_ERROR] + with suppress(Exception): + outcome[0] = _python_project_fields( + load_project_config(project_dir=project_dir), load_local_config(project_dir=project_dir) + ) + return outcome[0] + + +def native_project_outcome(*, project_dir: Path) -> object: + """Return the discovery fields the native reader reads, or its failure marker.""" + + parsed: dict[str, object] = json.loads(_native._oracle_project_config(str(project_dir))) + return {True: DEFERRED, False: parsed}["error" in parsed] + + +def deferred_valid_count(*, expected: list[object], actual: list[object]) -> int: + """Return how many documents Python accepts but the native reader defers back to Python.""" + + return sum( + map( + lambda python, native: native == DEFERRED and python not in UNCHECKED_PYTHON_OUTCOMES, + expected, + actual, + ) + ) + + +def _is_harmful(expected: object, actual: object) -> bool: + return actual != DEFERRED and actual != expected + + +def harmful_mismatches( + *, inputs: list[object], expected: list[object], actual: list[object] +) -> list[tuple[object, object, object]]: + """Return the first few cases where the native reader accepts a different result. + + A native failure defers the file to the Python loader, so only an accepted value that + differs from Python's outcome, including accepting what Python rejects, is harmful. + """ + + return list( + compress(zip(inputs, expected, actual, strict=True), map(_is_harmful, expected, actual)) + )[:3] diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py new file mode 100644 index 0000000000..243dd3840b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py @@ -0,0 +1,48 @@ +"""Compare the native project configuration reader with SQLBuild's Python loaders.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + deferred_valid_count, + harmful_mismatches, + native_project_outcome, + python_project_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="fixture and example projects", + pattern="**/sqlbuild_project.toml", + expected_minimum_files=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_projects_when_reading_natively_then_discovery_fields_match_python( + test_case: RepositoryFileOracleTestCase, +) -> None: + project_dirs: list[Path] = [path.parent for path in repository_files(pattern=test_case.pattern)] + + expected: list[object] = [python_project_outcome(project_dir=path) for path in project_dirs] + actual: list[object] = [native_project_outcome(project_dir=path) for path in project_dirs] + + assert len(project_dirs) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, project_dirs)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py new file mode 100644 index 0000000000..f44563fea8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py @@ -0,0 +1,128 @@ +"""Compare the native TOML loader with `tomllib` on seeded and repository documents.""" + +from __future__ import annotations + +import random +import time +from collections.abc import Callable +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + GeneratedDocumentOracleTestCase, + HostileDocumentOracleTestCase, + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + DEFERRED, + HOSTILE_TOML_DOCUMENTS, + TOML_DOCUMENT_GENERATORS, + deferred_valid_count, + harmful_mismatches, + native_toml_outcome, + python_toml_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedDocumentOracleTestCase( + description="tables, dotted keys, arrays of tables, strings, numbers and datetimes", + generator="toml", + seed=51, + case_count=5000, + expected_maximum_deferred=350, + ), + GeneratedDocumentOracleTestCase( + description="tables, arrays of tables, inline tables and dotted keys reopening namespaces", + generator="toml namespaces", + seed=52, + case_count=5000, + expected_maximum_deferred=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_toml_when_loading_natively_then_values_match_tomllib( + test_case: GeneratedDocumentOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + generate: Callable[[random.Random], str] = TOML_DOCUMENT_GENERATORS[test_case.generator] + texts: list[str] = [generate(rng) for _ in range(test_case.case_count)] + + expected: list[object] = [python_toml_outcome(text=text) for text in texts] + actual: list[object] = [native_toml_outcome(text=text) for text in texts] + + assert ( + deferred_valid_count(expected=expected, actual=actual) + <= test_case.expected_maximum_deferred + ) + assert harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="repository TOML files", pattern="**/*.toml", expected_minimum_files=8 + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_toml_files_when_loading_natively_then_values_match_tomllib( + test_case: RepositoryFileOracleTestCase, +) -> None: + paths: list[Path] = repository_files(pattern=test_case.pattern) + texts: list[str] = [path.read_text(encoding="utf-8") for path in paths] + + expected: list[object] = [python_toml_outcome(text=text) for text in texts] + actual: list[object] = [native_toml_outcome(text=text) for text in texts] + + assert len(paths) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, paths)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +@pytest.mark.parametrize( + "test_case", + [ + HostileDocumentOracleTestCase( + description="ten thousand nested arrays", document="deep arrays" + ), + HostileDocumentOracleTestCase( + description="ten thousand nested inline tables", document="deep inline tables" + ), + HostileDocumentOracleTestCase( + description="a dotted key with a hundred thousand segments", document="long dotted key" + ), + HostileDocumentOracleTestCase( + description="a table header with a hundred thousand segments", + document="long table header", + ), + ], + ids=lambda case: case.description, +) +def test_given_hostile_toml_when_loading_natively_then_reader_defers_quickly( + test_case: HostileDocumentOracleTestCase, +) -> None: + text: str = HOSTILE_TOML_DOCUMENTS[test_case.document]() + + started: float = time.perf_counter() + outcome: object = native_toml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (outcome == DEFERRED, elapsed < test_case.expected_maximum_seconds) == ( + test_case.expected_deferred, + True, + ), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py new file mode 100644 index 0000000000..132ce747ef --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py @@ -0,0 +1,177 @@ +"""Compare the native YAML loader with PyYAML `safe_load` on seeded and repository documents.""" + +from __future__ import annotations + +import random +import time +from collections.abc import Callable +from pathlib import Path + +import pytest +import yaml + +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + GeneratedDocumentOracleTestCase, + HostileDocumentOracleTestCase, + LargeDocumentOracleTestCase, + RepositoryFileOracleTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + DEFERRED, + HOSTILE_YAML_DOCUMENTS, + LARGE_YAML_DOCUMENTS, + YAML_DOCUMENT_GENERATORS, + deferred_valid_count, + expected_yaml_outcome, + harmful_mismatches, + native_yaml_outcome, + python_yaml_outcome, + repository_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedDocumentOracleTestCase( + description="near-miss plain, quoted and tagged scalars", + generator="scalars", + seed=41, + case_count=4000, + expected_maximum_deferred=10, + ), + GeneratedDocumentOracleTestCase( + description="anchors, aliases, merge keys, chomped block scalars and comments", + generator="anchors", + seed=42, + case_count=1500, + expected_maximum_deferred=20, + ), + GeneratedDocumentOracleTestCase( + description="documents written by PyYAML's safe dumper in random styles", + generator="dumped", + seed=43, + case_count=1500, + expected_maximum_deferred=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_yaml_when_loading_natively_then_values_match_both_pyyaml_loaders( + test_case: GeneratedDocumentOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + generate: Callable[[random.Random], str] = YAML_DOCUMENT_GENERATORS[test_case.generator] + texts: list[str] = [generate(rng) for _ in range(test_case.case_count)] + + expected: list[object] = [expected_yaml_outcome(text=text) for text in texts] + actual: list[object] = [native_yaml_outcome(text=text) for text in texts] + + assert ( + deferred_valid_count(expected=expected, actual=actual) + <= test_case.expected_maximum_deferred + ) + assert harmful_mismatches(inputs=list(texts), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryFileOracleTestCase( + description="repository YAML files", pattern="**/*.yml", expected_minimum_files=10 + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_yaml_files_when_loading_natively_then_values_match_both_loaders( + test_case: RepositoryFileOracleTestCase, +) -> None: + paths: list[Path] = repository_files(pattern=test_case.pattern) + texts: list[str] = [path.read_text(encoding="utf-8") for path in paths] + + expected: list[object] = [expected_yaml_outcome(text=text) for text in texts] + actual: list[object] = [native_yaml_outcome(text=text) for text in texts] + + assert len(paths) >= test_case.expected_minimum_files + assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred + assert harmful_mismatches( + inputs=list(map(str, paths)), expected=expected, actual=actual + ) == list(test_case.expected_mismatches) + + +@pytest.mark.parametrize( + "test_case", + [ + HostileDocumentOracleTestCase( + description="a ten thousand link alias chain", document="alias chain" + ), + HostileDocumentOracleTestCase( + description="billion laughs with twelve levels", document="billion laughs" + ), + HostileDocumentOracleTestCase( + description="merge keys doubling over thirty levels", document="merge chain" + ), + HostileDocumentOracleTestCase( + description="a hundred thousand nested flow sequences", document="deep flow" + ), + HostileDocumentOracleTestCase( + description="a hundred thousand nested block sequences", document="deep block" + ), + HostileDocumentOracleTestCase( + description="an integer beyond Python's string conversion limit", + document="long integer", + ), + ], + ids=lambda case: case.description, +) +def test_given_hostile_yaml_when_loading_natively_then_reader_defers_quickly( + test_case: HostileDocumentOracleTestCase, +) -> None: + text: str = HOSTILE_YAML_DOCUMENTS[test_case.document]() + + started: float = time.perf_counter() + outcome: object = native_yaml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (outcome == DEFERRED, elapsed < test_case.expected_maximum_seconds) == ( + test_case.expected_deferred, + True, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + LargeDocumentOracleTestCase( + description="a hundred thousand keys in one flow mapping line", document="flow mapping" + ), + LargeDocumentOracleTestCase( + description="two hundred thousand items in one flow sequence line", + document="flow sequence", + ), + LargeDocumentOracleTestCase( + description="fifty thousand anchored items in one flow sequence line", + document="anchored flow sequence", + ), + ], + ids=lambda case: case.description, +) +def test_given_large_single_line_yaml_when_loading_natively_then_time_is_linear_and_values_match( + test_case: LargeDocumentOracleTestCase, +) -> None: + text: str = LARGE_YAML_DOCUMENTS[test_case.document]() + expected: object = python_yaml_outcome(text=text, loader=yaml.CSafeLoader) + + started: float = time.perf_counter() + actual: object = native_yaml_outcome(text=text) + elapsed: float = time.perf_counter() - started + + assert (actual, elapsed < test_case.expected_maximum_seconds) == (expected, True), ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/helpers.py b/tests/integration/src/sqlbuild/compiler/helpers.py new file mode 100644 index 0000000000..67f935b588 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/helpers.py @@ -0,0 +1,301 @@ +"""Seeded generators and Python-side results shared by the native oracle tests.""" + +from __future__ import annotations + +import json +import math +import random +import struct +from collections.abc import Callable +from contextlib import suppress +from itertools import compress +from operator import ne +from pathlib import Path + +import orjson + +from sqlbuild import _native +from tests.integration.src.sqlbuild.compiler._test_types import JsonOracleTestCase + +NASTY_FLOATS: tuple[float, ...] = ( + 0.0, + -0.0, + math.nan, + math.inf, + -math.inf, + 5e-324, + -5e-324, + 2.2250738585072014e-308, + 1.7976931348623157e308, + 1e16, + 9999999999999998.0, + 1e15, + 1e-4, + 1e-5, + 9.999999999999999e-5, + 0.1, + 0.30000000000000004, + 1 / 3, + 2.0**53, + 2.0**53 + 2, + 1e22, + 1e-7, + 1.5e-6, + 100.0, + -123456.789, +) +NASTY_INTEGERS: tuple[int, ...] = ( + 0, + -1, + 2**31, + 2**53 + 1, + 2**63 - 1, + 2**63, + -(2**63), + -(2**63) - 1, + 2**64 - 1, + 2**64, + 10**40, + -(10**40), +) +CHARACTER_RANGES: tuple[tuple[int, int], ...] = ( + (0x20, 0x7E), + (0x20, 0x7E), + (0x20, 0x7E), + (0x00, 0x1F), + (0x7F, 0xA0), + (0xA1, 0x24F), + (0x2000, 0x206F), + (0x4E00, 0x4E80), + (0xE000, 0xE010), + (0xFFF0, 0xFFFF), + (0x1F600, 0x1F64F), + (0x10FFF0, 0x10FFFF), + (0x22, 0x22), + (0x5C, 0x5C), + (0x2028, 0x2029), + (0xFEFF, 0xFEFF), +) +LINE_BREAKS: tuple[str, ...] = ("\n", "\r\n", "\r", "\t", "\r\r\n") +INVALID_UTF8: tuple[bytes, ...] = (b"\xff", b"\xc3", b"\xed\xa0\x80", b"\xf4\x90\x80\x80") +MAX_JSON_DEPTH: int = 4 + + +def random_character(*, rng: random.Random) -> str: + """Return one code point from a range chosen to stress escaping and offsets.""" + + low, high = rng.choice(CHARACTER_RANGES) + return chr(rng.randint(low, high)) + + +def random_text(*, rng: random.Random, max_length: int) -> str: + """Return a string mixing ASCII, controls, non-BMP and other awkward code points.""" + + return "".join(random_character(rng=rng) for _ in range(rng.randint(0, max_length))) + + +def _float_from_bits(rng: random.Random) -> float: + return struct.unpack(" float: + mantissa: int = rng.randint(1, 10 ** rng.randint(1, 17)) + return float(f"{rng.choice('-+')}{mantissa}e{rng.randint(-340, 300)}") + + +FLOAT_GENERATORS: tuple[Callable[[random.Random], float], ...] = ( + lambda rng: rng.choice(NASTY_FLOATS), + _float_from_bits, + _float_from_decimal, +) +INTEGER_GENERATORS: tuple[Callable[[random.Random], int], ...] = ( + lambda rng: rng.choice(NASTY_INTEGERS) + rng.randint(-2, 2), + lambda rng: rng.randint(-(2 ** rng.randint(1, 140)), 2 ** rng.randint(1, 140)), +) + + +def random_float(*, rng: random.Random) -> float: + """Return a float from the nasty pool, random bits, or a random decimal.""" + + return rng.choice(FLOAT_GENERATORS)(rng) + + +def random_integer(*, rng: random.Random) -> int: + """Return an integer near the 32- and 64-bit boundaries or of arbitrary size.""" + + return rng.choice(INTEGER_GENERATORS)(rng) + + +def _json_list(rng: random.Random, depth: int) -> object: + return [random_json_value(rng=rng, depth=depth - 1) for _ in range(rng.randint(0, 4))] + + +def _json_dict(rng: random.Random, depth: int) -> object: + return { + random_text(rng=rng, max_length=6): random_json_value(rng=rng, depth=depth - 1) + for _ in range(rng.randint(0, 4)) + } + + +JSON_SCALAR_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + lambda rng, depth: None, + lambda rng, depth: rng.random() < 0.5, + lambda rng, depth: random_integer(rng=rng), + lambda rng, depth: random_float(rng=rng), + lambda rng, depth: random_text(rng=rng, max_length=12), +) +JSON_GENERATORS: tuple[Callable[[random.Random, int], object], ...] = ( + *JSON_SCALAR_GENERATORS, + _json_list, + _json_dict, + _json_list, + _json_dict, +) + + +def random_json_value(*, rng: random.Random, depth: int) -> object: + """Return a random JSON-compatible Python value, preferring containers above the leaves.""" + + available: int = len(JSON_SCALAR_GENERATORS) + 4 * min(depth, 1) + return rng.choice(JSON_GENERATORS[:available])(rng, depth) + + +def _float_bits(value: float) -> str: + return str(struct.unpack(" list[object]: + """Return the tagged form the native JSON oracle hook decodes.""" + + return TAGGERS[type(value)](value) + + +def python_json_text(*, value: object, test_case: JsonOracleTestCase) -> str: + """Return Python's serializer output, or the name of the failure it raises.""" + + outcome: list[str] = [f"error:{SERIALIZER_FAILURES[test_case.serializer]}"] + with suppress(ValueError, TypeError): + outcome[0] = PYTHON_SERIALIZERS[test_case.serializer](value, test_case) + return outcome[0] + + +def _stdlib_dumps(value: object, test_case: JsonOracleTestCase) -> str: + return json.dumps( + value, + indent=test_case.indent, + separators=test_case.separators, + ensure_ascii=test_case.ensure_ascii, + sort_keys=test_case.sort_keys, + allow_nan=test_case.allow_nan, + ) + + +PYTHON_SERIALIZERS: dict[str, Callable[[object, JsonOracleTestCase], str]] = { + "stdlib": _stdlib_dumps, + "orjson": lambda value, test_case: orjson.dumps(value, option=test_case.orjson_option).decode(), +} +SERIALIZER_FAILURES: dict[str, str] = {"stdlib": "NonFiniteFloat", "orjson": "IntegerOutOfRange"} + + +INDENT_TEXT: dict[type, Callable[[object], object]] = { + int: lambda width: " " * width, + str: lambda text: text, + type(None): lambda nothing: nothing, +} + + +def native_json_text(*, value: object, test_case: JsonOracleTestCase) -> str: + """Return the native emitter's output for the same value and configuration.""" + + default_separators: tuple[str, str] = ((", ", ": "), (",", ": "))[test_case.indent is not None] + item_separator, key_separator = test_case.separators or default_separators + indent: object = INDENT_TEXT[type(test_case.indent)](test_case.indent) + specs: dict[str, dict[str, object]] = { + "stdlib": { + "indent": indent, + "separators": {"item": item_separator, "key": key_separator}, + "ensure_ascii": test_case.ensure_ascii, + "sort_keys": test_case.sort_keys, + "allow_nan": test_case.allow_nan, + }, + "orjson": { + "orjson": { + "indent_2": bool(test_case.orjson_option & orjson.OPT_INDENT_2), + "sort_keys": bool(test_case.orjson_option & orjson.OPT_SORT_KEYS), + } + }, + } + spec: dict[str, object] = specs[test_case.serializer] + return _native._oracle_json_dumps(json.dumps(spec), json.dumps(tagged(value))) + + +def mismatches( + *, inputs: list[object], expected: list[object], actual: list[object] +) -> list[tuple[object, object, object]]: + """Return the first few `(input, expected, actual)` triples that differ.""" + + return list(compress(zip(inputs, expected, actual, strict=True), map(ne, expected, actual)))[:3] + + +def authored_bytes(*, rng: random.Random) -> bytes: + """Return valid UTF-8 text with every newline convention, tabs and non-ASCII.""" + + parts: list[str] = [ + random_text(rng=rng, max_length=8) + rng.choice(LINE_BREAKS) + for _ in range(rng.randint(0, 12)) + ] + return "".join(parts).encode("utf-8") + + +def invalid_authored_bytes(*, rng: random.Random) -> bytes: + """Return authored bytes with one invalid UTF-8 sequence inserted.""" + + data: bytes = authored_bytes(rng=rng) + cut: int = rng.randint(0, len(data)) + return data[:cut] + rng.choice(INVALID_UTF8) + data[cut:] + + +def python_positions(*, text: str, char_offsets: list[int]) -> list[tuple[int, int, int]]: + """Return `(char_offset, line, column)` with SQLBuild's Python line and column arithmetic.""" + + return [ + ( + offset, + text.count("\n", 0, offset) + 1, + offset - (text.rfind("\n", 0, offset) + 1) + 1, + ) + for offset in char_offsets + ] + + +def random_char_offsets(*, rng: random.Random, text: str) -> list[int]: + """Return sorted random code-point offsets into `text`, including its end.""" + + return sorted(rng.randint(0, len(text)) for _ in range(8)) + + +def utf8_offsets(*, text: str, char_offsets: list[int]) -> list[int]: + """Return the UTF-8 byte offset of each code-point offset into `text`.""" + + return [len(text[:offset].encode("utf-8")) for offset in char_offsets] + + +def python_read_text(*, path: Path, data: bytes) -> str: + """Write `data` and read it back the way SQLBuild reads authored files.""" + + path.write_bytes(data) + return path.read_text(encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py new file mode 100644 index 0000000000..749b212c47 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/test_native_json_oracle.py @@ -0,0 +1,149 @@ +"""Compare the native JSON emitter with Python's serializers on seeded generated values.""" + +from __future__ import annotations + +import random + +import orjson +import pytest + +from tests.integration.src.sqlbuild.compiler._test_types import ( + JsonOracleTestCase, + OrjsonFloatLayoutTestCase, +) +from tests.integration.src.sqlbuild.compiler.helpers import ( + MAX_JSON_DEPTH, + mismatches, + native_json_text, + python_json_text, + random_json_value, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JsonOracleTestCase( + description="json.dumps defaults", serializer="stdlib", seed=11, case_count=1500 + ), + JsonOracleTestCase( + description="json.dumps indent=2 for manifests and DAG output", + serializer="stdlib", + seed=12, + case_count=1500, + indent=2, + ), + JsonOracleTestCase( + description="json.dumps compact sorted ASCII for identities and caches", + serializer="stdlib", + seed=13, + case_count=1500, + separators=(",", ":"), + sort_keys=True, + ), + JsonOracleTestCase( + description="json.dumps Unicode sorted with indent", + serializer="stdlib", + seed=14, + case_count=1500, + indent=2, + ensure_ascii=False, + sort_keys=True, + ), + JsonOracleTestCase( + description="json.dumps tab indent with custom separators", + serializer="stdlib", + seed=15, + case_count=1500, + indent="\t", + separators=(", ", " = "), + ), + JsonOracleTestCase( + description="json.dumps allow_nan=False rejects non-finite floats", + serializer="stdlib", + seed=16, + case_count=1500, + separators=(",", ":"), + allow_nan=False, + ), + JsonOracleTestCase( + description="orjson defaults", serializer="orjson", seed=21, case_count=1500 + ), + JsonOracleTestCase( + description="orjson OPT_INDENT_2 for the compile payload", + serializer="orjson", + seed=22, + case_count=1500, + orjson_option=orjson.OPT_INDENT_2, + ), + JsonOracleTestCase( + description="orjson OPT_SORT_KEYS", + serializer="orjson", + seed=23, + case_count=1500, + orjson_option=orjson.OPT_SORT_KEYS, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_values_when_dumping_natively_then_bytes_match_python_serializer( + test_case: JsonOracleTestCase, +) -> None: + rng: random.Random = random.Random(test_case.seed) + values: list[object] = [ + random_json_value(rng=rng, depth=MAX_JSON_DEPTH) for _ in range(test_case.case_count) + ] + + expected: list[object] = [ + python_json_text(value=value, test_case=test_case) for value in values + ] + actual: list[object] = [native_json_text(value=value, test_case=test_case) for value in values] + + assert mismatches(inputs=values, expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [ + OrjsonFloatLayoutTestCase( + description="positive exponents carry a plus sign from orjson 3.11.7", + value=1e16, + expected_text="1e+16", + ), + OrjsonFloatLayoutTestCase( + description="large positive exponents carry a plus sign", + value=1.5e300, + expected_text="1.5e+300", + ), + OrjsonFloatLayoutTestCase( + description="negative exponents keep their sign without padding", + value=1e-6, + expected_text="1e-6", + ), + OrjsonFloatLayoutTestCase( + description="small floats stay fixed down to five decimal places", + value=1e-5, + expected_text="0.00001", + ), + ], + ids=lambda case: case.description, +) +def test_given_installed_orjson_when_dumping_exponent_floats_then_text_matches_native_layout( + test_case: OrjsonFloatLayoutTestCase, +) -> None: + orjson_case: JsonOracleTestCase = JsonOracleTestCase( + description=test_case.description, serializer="orjson", seed=0, case_count=1 + ) + + installed: str = orjson.dumps(test_case.value).decode() + native: str = native_json_text(value=test_case.value, test_case=orjson_case) + + assert (installed, native) == (test_case.expected_text, test_case.expected_text), ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py b/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py new file mode 100644 index 0000000000..dcde8880bb --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/test_native_text_oracle.py @@ -0,0 +1,86 @@ +"""Compare native text decoding and positions with Python's read_text and column arithmetic.""" + +from __future__ import annotations + +import random +from pathlib import Path + +import pytest + +from sqlbuild import _native +from tests.integration.src.sqlbuild.compiler._test_types import TextOracleTestCase +from tests.integration.src.sqlbuild.compiler.helpers import ( + authored_bytes, + invalid_authored_bytes, + mismatches, + python_positions, + python_read_text, + random_char_offsets, + utf8_offsets, +) + + +@pytest.mark.parametrize( + "test_case", + [ + TextOracleTestCase( + description="CRLF, CR, tabs and non-ASCII before every offset", seed=31, case_count=400 + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_files_when_decoding_natively_then_text_and_positions_match_python( + test_case: TextOracleTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + files: list[bytes] = [authored_bytes(rng=rng) for _ in range(test_case.case_count)] + texts: list[str] = [python_read_text(path=tmp_path / "model.sql", data=data) for data in files] + char_offsets: list[list[int]] = [random_char_offsets(rng=rng, text=text) for text in texts] + byte_offsets: list[list[int]] = [ + utf8_offsets(text=text, char_offsets=offsets) + for text, offsets in zip(texts, char_offsets, strict=True) + ] + + expected: list[object] = [ + (text, python_positions(text=text, char_offsets=offsets)) + for text, offsets in zip(texts, char_offsets, strict=True) + ] + actual: list[object] = [ + _native._oracle_text_positions(data, offsets) + for data, offsets in zip(files, byte_offsets, strict=True) + ] + + assert mismatches(inputs=list(files), expected=expected, actual=actual) == list( + test_case.expected_mismatches + ) + + +@pytest.mark.parametrize( + "test_case", + [TextOracleTestCase(description="one invalid UTF-8 sequence", seed=32, case_count=100)], + ids=lambda case: case.description, +) +def test_given_invalid_utf8_when_decoding_natively_then_failure_offset_matches_python( + test_case: TextOracleTestCase, tmp_path: Path +) -> None: + rng: random.Random = random.Random(test_case.seed) + path: Path = tmp_path / "model.sql" + files: list[bytes] = [invalid_authored_bytes(rng=rng) for _ in range(test_case.case_count)] + python_offsets: list[object] = [] + native_offsets: list[object] = [] + + for data in files: + with pytest.raises(UnicodeDecodeError) as python_error: + python_read_text(path=path, data=data) + with pytest.raises(ValueError) as native_error: + _native._oracle_text_positions(data, []) + python_offsets.append(python_error.value.start) + native_offsets.append(int(str(native_error.value))) + + assert mismatches(inputs=list(files), expected=python_offsets, actual=native_offsets) == list( + test_case.expected_mismatches + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/uv.lock b/uv.lock index c66af47428..ba6f77008a 100644 --- a/uv.lock +++ b/uv.lock @@ -4217,7 +4217,7 @@ requires-dist = [ { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.37.0" }, { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.0.0" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, - { name = "orjson", specifier = ">=3.10.0" }, + { name = "orjson", specifier = ">=3.11.7" }, { name = "polyglot-sql-chio", specifier = "==0.13.8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2.0" }, { name = "pydantic-settings", specifier = ">=2,<3" },