diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index ffe07b885..739c7317a 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -1,8 +1,19 @@ name: Website -# Builds the site and checks its links and branding. Manual only, and it never deploys: the site is -# not public yet. Add push and pull_request triggers, and a deploy job, once publishing is approved. +# Builds the playground engine and the site, and checks them. It runs on pull requests that touch +# them and on demand, never on push to main, and it never deploys: the site is not public yet. +# Add a deploy job only once publishing is approved. on: + pull_request: + paths: + - website/** + - crates/sqlquack-core/** + - crates/sqlquack-duckdb/** + - crates/sqlquack-rules/** + - crates/sqlquack-semantics/** + - crates/sqlquack-snowflake/** + - crates/sqlquack-syntax/** + - .github/workflows/website.yml workflow_dispatch: permissions: @@ -14,7 +25,7 @@ concurrency: jobs: build: - name: Build the website + name: Build the engine and the website runs-on: ubuntu-24.04 defaults: run: @@ -28,12 +39,38 @@ jobs: cache: npm cache-dependency-path: website/package-lock.json + - name: Install the pinned Rust toolchain with the wasm32 target + working-directory: . + run: | + rustup toolchain install + rustup target add wasm32-unknown-unknown + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: website/wasm + + - uses: taiki-e/install-action@v2 + with: + tool: wasm-bindgen-cli@0.2.130 + - name: Check the diagnostics reference matches the registry run: python3 scripts/generate_diagnostics.py --check + - name: Lint the engine + working-directory: website/wasm + run: | + cargo fmt --check + cargo clippy --locked --target wasm32-unknown-unknown --release -- -D warnings + - name: Install dependencies run: npm ci + - name: Build the engine + run: ./scripts/build_engine.sh + + - name: Test the engine and its spans + run: npm run -s test:engine + - name: Build run: npm run build diff --git a/Cargo.lock b/Cargo.lock index 0fc18581d..149ecdafe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,6 +1427,7 @@ dependencies = [ "sqlquack-core", "sqlquack-evidence", "sqlquack-policy", + "sqlquack-rules", "sqlquack-syntax", "toml", "walkdir", @@ -1452,6 +1453,7 @@ dependencies = [ "serde_json", "sqlquack-core", "sqlquack-duckdb", + "sqlquack-policy", "sqlquack-semantics", "sqlquack-snowflake", "sqlquack-syntax", diff --git a/crates/sqlquack-policy/src/policy/models.rs b/crates/sqlquack-policy/src/policy/models.rs index afc86a1bd..62dd9ba2d 100644 --- a/crates/sqlquack-policy/src/policy/models.rs +++ b/crates/sqlquack-policy/src/policy/models.rs @@ -1,9 +1,13 @@ //! The policy schema. Every struct rejects unknown keys. use std::collections::BTreeMap; +use std::str::FromStr; use serde::{Deserialize, Serialize}; +use crate::policy::_helpers::parsing::parse_policy; +use crate::policy::errors::PolicyError; + /// The whole policy file. #[derive(Clone, Debug, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] @@ -216,3 +220,12 @@ pub enum PatternArgument { /// A lower-case name bound by the first match. Variable(String), } + +/// A policy file's text, validated as `load_policy` validates a file; for callers without files. +impl FromStr for Policy { + type Err = PolicyError; + + fn from_str(text: &str) -> Result { + parse_policy(text) + } +} diff --git a/crates/sqlquack-polyglot-check/Cargo.toml b/crates/sqlquack-polyglot-check/Cargo.toml index eadac920e..eb69fda55 100644 --- a/crates/sqlquack-polyglot-check/Cargo.toml +++ b/crates/sqlquack-polyglot-check/Cargo.toml @@ -22,6 +22,7 @@ sqlquack.workspace = true sqlquack-core.workspace = true sqlquack-evidence.workspace = true sqlquack-policy.workspace = true +sqlquack-rules.workspace = true sqlquack-syntax.workspace = true toml.workspace = true walkdir.workspace = true diff --git a/crates/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs b/crates/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs index 25b149710..ee9df5438 100644 --- a/crates/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs +++ b/crates/sqlquack-polyglot-check/src/corpora/main/case_corpus.rs @@ -2,12 +2,12 @@ use std::path::Path; -use sqlquack::analysis::main::conversion_options::conversion_options; use sqlquack::analysis::models::PipelineConverter; use sqlquack_evidence::cases::main::load_cases::load_cases; use sqlquack_evidence::verify::models::Conversion; use sqlquack_evidence::verify::types::CaseConverter; use sqlquack_policy::policy::main::load_policy::load_policy; +use sqlquack_rules::session::main::conversion_options::conversion_options; use sqlquack_syntax::Dialect; use crate::corpora::models::CorpusEntry; diff --git a/crates/sqlquack-rules/Cargo.toml b/crates/sqlquack-rules/Cargo.toml index f64e6a2b6..19a868571 100644 --- a/crates/sqlquack-rules/Cargo.toml +++ b/crates/sqlquack-rules/Cargo.toml @@ -20,6 +20,7 @@ serde.workspace = true serde_json.workspace = true sqlquack-core.workspace = true sqlquack-duckdb.workspace = true +sqlquack-policy.workspace = true sqlquack-semantics.workspace = true sqlquack-snowflake.workspace = true sqlquack-syntax.workspace = true diff --git a/crates/sqlquack-rules/src/conversion/main/mod.rs b/crates/sqlquack-rules/src/conversion/main/mod.rs index 2ca96312e..df305206a 100644 --- a/crates/sqlquack-rules/src/conversion/main/mod.rs +++ b/crates/sqlquack-rules/src/conversion/main/mod.rs @@ -4,3 +4,4 @@ pub mod convert_statement; pub mod override_macro; pub mod override_macro_name; pub mod quoted_exclusions; +pub mod within_budget; diff --git a/crates/sqlquack-rules/src/conversion/main/within_budget.rs b/crates/sqlquack-rules/src/conversion/main/within_budget.rs new file mode 100644 index 000000000..8750b73bc --- /dev/null +++ b/crates/sqlquack-rules/src/conversion/main/within_budget.rs @@ -0,0 +1,43 @@ +//! `within_budget`: a conversion, or a refusal when converting it outran its time or size budget. + +use std::time::Duration; + +use sqlquack_core::refusals::models::{Refusal, RefusalCategory}; + +use crate::conversion::models::{ConversionOptions, Converted}; + +/// `converted`, or a refusal when converting it outran the time or size budget. +pub fn within_budget( + converted: Converted, + (source, elapsed): (&str, Duration), + options: &ConversionOptions, +) -> Result { + let budget = options.budget; + let emitted = converted.sql.len() as u64; + let allowed = budget + .minimum_bytes + .max((source.len() as u64).saturating_mul(budget.size_ratio)); + let reason = if elapsed > Duration::from_millis(budget.milliseconds) { + format!( + "converting the statement took {} ms, beyond the budget of {} ms", + elapsed.as_millis(), + budget.milliseconds + ) + } else if emitted > allowed { + format!( + "the DuckDB SQL is {emitted} bytes for {} bytes of source, beyond the budget of \ + {allowed} bytes ({} times the source, at least {})", + source.len(), + budget.size_ratio, + budget.minimum_bytes + ) + } else { + return Ok(converted); + }; + Err(Refusal { + construct: None, + location: None, + category: RefusalCategory::ConversionBudget, + reason, + }) +} diff --git a/crates/sqlquack-rules/src/lib.rs b/crates/sqlquack-rules/src/lib.rs index 03094daff..4c23bf847 100644 --- a/crates/sqlquack-rules/src/lib.rs +++ b/crates/sqlquack-rules/src/lib.rs @@ -4,3 +4,4 @@ pub mod catalogue; pub mod conversion; +pub mod session; diff --git a/crates/sqlquack-rules/src/session/_helpers/mod.rs b/crates/sqlquack-rules/src/session/_helpers/mod.rs new file mode 100644 index 000000000..c2cfe4b6c --- /dev/null +++ b/crates/sqlquack-rules/src/session/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod session_profiles; diff --git a/crates/sqlquack/src/analysis/_helpers/session_profiles.rs b/crates/sqlquack-rules/src/session/_helpers/session_profiles.rs similarity index 100% rename from crates/sqlquack/src/analysis/_helpers/session_profiles.rs rename to crates/sqlquack-rules/src/session/_helpers/session_profiles.rs diff --git a/crates/sqlquack-rules/src/session/constants.rs b/crates/sqlquack-rules/src/session/constants.rs new file mode 100644 index 000000000..96be33069 --- /dev/null +++ b/crates/sqlquack-rules/src/session/constants.rs @@ -0,0 +1,4 @@ +//! Session constants. + +/// The profile name reported when the built-in default stands in for a policy. +pub const BUILTIN_PROFILE: &str = "the built-in default"; diff --git a/crates/sqlquack/src/analysis/main/confirm_utc_session.rs b/crates/sqlquack-rules/src/session/main/confirm_utc_session.rs similarity index 83% rename from crates/sqlquack/src/analysis/main/confirm_utc_session.rs rename to crates/sqlquack-rules/src/session/main/confirm_utc_session.rs index a62ad6271..7eaf74126 100644 --- a/crates/sqlquack/src/analysis/main/confirm_utc_session.rs +++ b/crates/sqlquack-rules/src/session/main/confirm_utc_session.rs @@ -1,8 +1,8 @@ //! `confirm_utc_session`: conversion options trusting the caller's word that sessions run in UTC. +use crate::conversion::constants::SESSION_TIMEZONE_UTC; +use crate::conversion::models::{AllowedDivergence, ConversionOptions}; use sqlquack_policy::policy::models::Policy; -use sqlquack_rules::conversion::constants::SESSION_TIMEZONE_UTC; -use sqlquack_rules::conversion::models::{AllowedDivergence, ConversionOptions}; /// `options` relying on `session_timezone_utc` in any mode: the caller confirmed UTC sessions. pub fn confirm_utc_session(mut options: ConversionOptions, policy: &Policy) -> ConversionOptions { diff --git a/crates/sqlquack/src/analysis/main/conversion_options.rs b/crates/sqlquack-rules/src/session/main/conversion_options.rs similarity index 92% rename from crates/sqlquack/src/analysis/main/conversion_options.rs rename to crates/sqlquack-rules/src/session/main/conversion_options.rs index 18baf2375..9113cda85 100644 --- a/crates/sqlquack/src/analysis/main/conversion_options.rs +++ b/crates/sqlquack-rules/src/session/main/conversion_options.rs @@ -1,7 +1,7 @@ //! `conversion_options`: the known divergences the policy allows in one mode, and its budget. +use crate::conversion::models::{AllowedDivergence, ConversionBudget, ConversionOptions}; use sqlquack_policy::policy::models::{DivergenceMode, Policy}; -use sqlquack_rules::conversion::models::{AllowedDivergence, ConversionBudget, ConversionOptions}; /// The divergences `policy` converts with a warning, in strict mode or by default. pub fn conversion_options(policy: &Policy, strict: bool) -> ConversionOptions { diff --git a/crates/sqlquack-rules/src/session/main/mod.rs b/crates/sqlquack-rules/src/session/main/mod.rs new file mode 100644 index 000000000..eee42cc48 --- /dev/null +++ b/crates/sqlquack-rules/src/session/main/mod.rs @@ -0,0 +1,4 @@ +pub mod confirm_utc_session; +pub mod conversion_options; +pub mod parse_session; +pub mod session_conversion_options; diff --git a/crates/sqlquack/src/analysis/main/parse_session.rs b/crates/sqlquack-rules/src/session/main/parse_session.rs similarity index 90% rename from crates/sqlquack/src/analysis/main/parse_session.rs rename to crates/sqlquack-rules/src/session/main/parse_session.rs index 3d204ea12..fdb2c41ad 100644 --- a/crates/sqlquack/src/analysis/main/parse_session.rs +++ b/crates/sqlquack-rules/src/session/main/parse_session.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; /// `NAME=VALUE` settings as upper-case names to values; empty or conflicting names are an error. -pub(crate) fn parse_session(settings: &[String]) -> Result, String> { +pub fn parse_session(settings: &[String]) -> Result, String> { let mut parameters: BTreeMap = BTreeMap::new(); for setting in settings { let (name, value) = setting diff --git a/crates/sqlquack/src/analysis/main/session_conversion_options.rs b/crates/sqlquack-rules/src/session/main/session_conversion_options.rs similarity index 82% rename from crates/sqlquack/src/analysis/main/session_conversion_options.rs rename to crates/sqlquack-rules/src/session/main/session_conversion_options.rs index 3a8b25d55..a7583d7f9 100644 --- a/crates/sqlquack/src/analysis/main/session_conversion_options.rs +++ b/crates/sqlquack-rules/src/session/main/session_conversion_options.rs @@ -2,17 +2,17 @@ use std::collections::BTreeMap; +use crate::conversion::constants::SESSION_TIMEZONE_UTC; +use crate::conversion::models::ConversionOptions; use sqlquack_policy::policy::constants::{ DEFAULT_SESSION_PROFILE, QUOTED_IDENTIFIERS_IGNORE_CASE, SESSION_TIME_ZONE, TIMEZONE_PARAMETER, }; use sqlquack_policy::policy::models::{Policy, SessionValue}; -use sqlquack_rules::conversion::constants::SESSION_TIMEZONE_UTC; -use sqlquack_rules::conversion::models::ConversionOptions; -use crate::analysis::_helpers::session_profiles::{matching_profile, value_text}; -use crate::analysis::constants::BUILTIN_PROFILE; -use crate::analysis::main::confirm_utc_session::confirm_utc_session; -use crate::analysis::main::conversion_options::conversion_options; +use crate::session::_helpers::session_profiles::{matching_profile, value_text}; +use crate::session::constants::BUILTIN_PROFILE; +use crate::session::main::confirm_utc_session::confirm_utc_session; +use crate::session::main::conversion_options::conversion_options; /// Options for a session with `parameters`, under the recorded profile they match, and its name. pub fn session_conversion_options( diff --git a/crates/sqlquack-rules/src/session/mod.rs b/crates/sqlquack-rules/src/session/mod.rs new file mode 100644 index 000000000..2af0fc740 --- /dev/null +++ b/crates/sqlquack-rules/src/session/mod.rs @@ -0,0 +1,5 @@ +//! A caller's policy and Snowflake session parameters as conversion options. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlquack-semantics/src/types/_helpers/mod.rs b/crates/sqlquack-semantics/src/types/_helpers/mod.rs new file mode 100644 index 000000000..d3025096b --- /dev/null +++ b/crates/sqlquack-semantics/src/types/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod schema_file; diff --git a/crates/sqlquack-semantics/src/types/_helpers/schema_file.rs b/crates/sqlquack-semantics/src/types/_helpers/schema_file.rs new file mode 100644 index 000000000..23d54969a --- /dev/null +++ b/crates/sqlquack-semantics/src/types/_helpers/schema_file.rs @@ -0,0 +1,181 @@ +//! The schema file as written, and its columns and functions typed. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use sqlquack_core::provenance::models::ProvenanceLog; +use sqlquack_snowflake::typing::main::cast_target::cast_target; +use sqlquack_snowflake::typing::models::SnowflakeType; + +use crate::types::constants::DECLARABLE_OFFSETS; +use crate::types::models::{ + ColumnSpec, FunctionLanguage, FunctionReturns, FunctionSpec, FunctionTraits, NullInput, + Volatility, +}; + +/// Columns and parameters as the schema file writes them. +pub(crate) type RawColumns = Vec; + +/// A name and Snowflake type pair, or an object that may also mark a VARIANT's typed leaves. +#[derive(Deserialize)] +#[serde(untagged)] +pub(crate) enum RawColumn { + Pair(String, String), + Described(RawDescribedColumn), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawDescribedColumn { + name: String, + #[serde(rename = "type")] + type_name: String, + #[serde(default)] + typed_leaves: bool, + /// The offset every value of a TIMESTAMP_TZ source column holds: `+00:00` or `UTC`. + #[serde(default)] + offset: Option, +} + +/// The schema file: relations and functions, or relations alone as the top-level map. +#[derive(Deserialize)] +#[serde(untagged)] +pub(crate) enum RawSchema { + Sections(RawSections), + Relations(BTreeMap), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawSections { + #[serde(default)] + pub(crate) relations: BTreeMap, + #[serde(default)] + pub(crate) functions: BTreeMap, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawFunction { + language: RawLanguage, + #[serde(default)] + arguments: RawColumns, + returns: RawReturns, + #[serde(default)] + volatility: Volatility, + #[serde(default)] + null_input: NullInput, + #[serde(default)] + vectorized: bool, +} + +#[derive(Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum RawLanguage { + Sql, + Python, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub(crate) enum RawReturns { + Scalar(String), + Table { table: RawColumns }, +} + +pub(crate) fn typed_function( + function: RawFunction, + log: &ProvenanceLog, +) -> Result, String> { + let returns = match function.returns { + RawReturns::Scalar(type_name) => match cast_target(&type_name, log) { + Ok(target) => FunctionReturns::Scalar(target.data_type), + Err(_) => return Ok(None), + }, + RawReturns::Table { table } => match typed_columns(table, None, log)? { + Some(columns) => FunctionReturns::Table(columns), + None => return Ok(None), + }, + }; + let Some(arguments) = typed_columns(function.arguments, None, log)? else { + return Ok(None); + }; + Ok(Some(FunctionSpec { + language: match function.language { + RawLanguage::Sql => FunctionLanguage::Sql, + RawLanguage::Python => FunctionLanguage::Python, + }, + arguments, + returns, + traits: FunctionTraits { + volatility: function.volatility, + null_input: function.null_input, + vectorized: function.vectorized, + }, + })) +} + +/// The columns of `relation`, or of a function when None; None when a type is unknown. +pub(crate) fn typed_columns( + columns: RawColumns, + relation: Option<&str>, + log: &ProvenanceLog, +) -> Result>, String> { + let mut typed: Vec = Vec::with_capacity(columns.len()); + for column in columns { + let (name, type_name, typed_leaves, offset) = match column { + RawColumn::Pair(name, type_name) => (name, type_name, false, None), + RawColumn::Described(column) => ( + column.name, + column.type_name, + column.typed_leaves, + column.offset, + ), + }; + let Ok(target) = cast_target(&type_name, log) else { + return Ok(None); + }; + let declared = match offset { + None => Vec::new(), + Some(offset) => vec![declared_offset( + (relation, &name), + target.data_type, + &offset, + )?], + }; + typed.push(ColumnSpec { + typed_leaves, + utc_offset: !declared.is_empty(), + declared_offsets: declared, + ..ColumnSpec::new(name, target.data_type) + }); + } + Ok(Some(typed)) +} + +/// The label of a relation column declared to hold the UTC offset; any other declaration fails. +fn declared_offset( + (relation, name): (Option<&str>, &str), + data_type: SnowflakeType, + offset: &str, +) -> Result { + let Some(relation) = relation else { + return Err(format!( + "{name}: an offset may be declared only for a relation's column" + )); + }; + if !matches!(data_type, SnowflakeType::TimestampTz { .. }) { + return Err(format!( + "{relation}.{name}: an offset may be declared only for a TIMESTAMP_TZ column, not {data_type}" + )); + } + if !DECLARABLE_OFFSETS + .iter() + .any(|declarable| declarable.eq_ignore_ascii_case(offset)) + { + return Err(format!( + "{relation}.{name}: the declared offset `{offset}` is not supported; only +00:00 (UTC) is" + )); + } + Ok(format!("{relation}.{name}")) +} diff --git a/crates/sqlquack-semantics/src/types/constants.rs b/crates/sqlquack-semantics/src/types/constants.rs new file mode 100644 index 000000000..4777c514e --- /dev/null +++ b/crates/sqlquack-semantics/src/types/constants.rs @@ -0,0 +1,4 @@ +//! Type constants. + +/// The offsets a schema may declare for a TIMESTAMP_TZ source column, all meaning +00:00. +pub const DECLARABLE_OFFSETS: &[&str] = &["+00:00", "UTC"]; diff --git a/crates/sqlquack-semantics/src/types/errors.rs b/crates/sqlquack-semantics/src/types/errors.rs new file mode 100644 index 000000000..ddddc149e --- /dev/null +++ b/crates/sqlquack-semantics/src/types/errors.rs @@ -0,0 +1,21 @@ +//! Schema file failures. + +use std::fmt::{self, Display}; + +/// Why a schema file's text is not a schema. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SchemaError { + /// The text is not JSON of the schema file's shape. + Shape(String), + /// A declaration the shape allows but SQLQuack does not, such as an unsupported offset. + Declaration(String), +} + +impl Display for SchemaError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Shape(error) => write!(formatter, "not a schema file: {error}"), + Self::Declaration(reason) => formatter.write_str(reason), + } + } +} diff --git a/crates/sqlquack-semantics/src/types/main/mod.rs b/crates/sqlquack-semantics/src/types/main/mod.rs index 086eaba9e..984af3e17 100644 --- a/crates/sqlquack-semantics/src/types/main/mod.rs +++ b/crates/sqlquack-semantics/src/types/main/mod.rs @@ -1,3 +1,4 @@ +pub mod parse_schema; pub mod schema_columns; pub mod utc_kept; pub mod zoned_in_utc; diff --git a/crates/sqlquack-semantics/src/types/main/parse_schema.rs b/crates/sqlquack-semantics/src/types/main/parse_schema.rs new file mode 100644 index 000000000..7842d05c6 --- /dev/null +++ b/crates/sqlquack-semantics/src/types/main/parse_schema.rs @@ -0,0 +1,34 @@ +//! `parse_schema`: a schema file's text as typed relations and project function signatures. + +use std::collections::BTreeMap; + +use sqlquack_core::provenance::models::ProvenanceLog; + +use crate::types::_helpers::schema_file::{RawSchema, typed_columns, typed_function}; +use crate::types::errors::SchemaError; +use crate::types::models::Schema; + +/// The schema `text` declares; a relation or function using a type SQLQuack does not know is left out. +pub fn parse_schema(text: &str) -> Result { + let raw: RawSchema = + serde_json::from_str(text).map_err(|error| SchemaError::Shape(error.to_string()))?; + let (relations, functions) = match raw { + RawSchema::Sections(sections) => (sections.relations, sections.functions), + RawSchema::Relations(relations) => (relations, BTreeMap::new()), + }; + let log = ProvenanceLog::default(); + let mut schema = Schema::default(); + for (relation, columns) in relations { + if let Some(typed) = + typed_columns(columns, Some(&relation), &log).map_err(SchemaError::Declaration)? + { + schema.relations.insert(relation, typed); + } + } + for (name, function) in functions { + if let Some(typed) = typed_function(function, &log).map_err(SchemaError::Declaration)? { + schema.functions.insert(name, typed); + } + } + Ok(schema) +} diff --git a/crates/sqlquack-semantics/src/types/mod.rs b/crates/sqlquack-semantics/src/types/mod.rs index 3c0912363..67bfd184f 100644 --- a/crates/sqlquack-semantics/src/types/mod.rs +++ b/crates/sqlquack-semantics/src/types/mod.rs @@ -1,5 +1,8 @@ //! Column and value types, with the marks tracked for VARIANT values. +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlquack/src/analysis/_helpers/mod.rs b/crates/sqlquack/src/analysis/_helpers/mod.rs index f831c2623..a146a5de7 100644 --- a/crates/sqlquack/src/analysis/_helpers/mod.rs +++ b/crates/sqlquack/src/analysis/_helpers/mod.rs @@ -6,5 +6,4 @@ pub(crate) mod override_calls; pub(crate) mod override_definition; pub(crate) mod placed; pub(crate) mod python_signature; -pub(crate) mod session_profiles; pub(crate) mod volatile_calls; diff --git a/crates/sqlquack/src/analysis/constants.rs b/crates/sqlquack/src/analysis/constants.rs index 5805fa218..de8af4eab 100644 --- a/crates/sqlquack/src/analysis/constants.rs +++ b/crates/sqlquack/src/analysis/constants.rs @@ -9,12 +9,6 @@ pub const SCALAR_RESULT_COLUMN: &str = "RESULT"; /// Settings every DuckDB session running converted SQL must have: the proofs hold only in UTC. pub const REQUIRED_DUCKDB_SETTINGS: &[(&str, &str)] = &[("TimeZone", "UTC")]; -/// The profile name reported when the built-in default stands in for a policy. -pub const BUILTIN_PROFILE: &str = "the built-in default"; - -/// The offsets a schema may declare for a TIMESTAMP_TZ source column, all meaning +00:00. -pub const DECLARABLE_OFFSETS: &[&str] = &["+00:00", "UTC"]; - /// How a Python function's scalar values cross: VARCHAR, NUMBER, FLOAT and BOOLEAN as recorded. pub const SCALAR_ABI: &str = "scalar-v1"; /// How a Python function's VARIANT result crosses: JSON text written by SQLQuack's runtime. diff --git a/crates/sqlquack/src/analysis/main/analyze.rs b/crates/sqlquack/src/analysis/main/analyze.rs index 88661d304..6a63d04dd 100644 --- a/crates/sqlquack/src/analysis/main/analyze.rs +++ b/crates/sqlquack/src/analysis/main/analyze.rs @@ -3,16 +3,17 @@ use sqlquack_core::classification::main::classify::classify; use sqlquack_core::parsing::main::parse_source::parse_source; use sqlquack_syntax::Expression; -use std::time::{Duration, Instant}; +use std::time::Instant; use sqlquack_core::parsing::models::{SourceDialect, SourceLines}; use sqlquack_core::refusals::main::refuse_unparsed::refuse_unparsed; use sqlquack_core::refusals::types::ConstructPolicy; use sqlquack_core::provenance::models::ProvenanceLog; -use sqlquack_core::refusals::models::{Refusal, RefusalCategory}; +use sqlquack_core::refusals::models::Refusal; use sqlquack_rules::conversion::main::convert_statement::convert_statement; use sqlquack_rules::conversion::main::quoted_exclusions::quoted_exclusions; +use sqlquack_rules::conversion::main::within_budget::within_budget; use sqlquack_rules::conversion::models::{ConversionOptions, Converted}; use sqlquack_semantics::types::models::Schema; @@ -89,39 +90,3 @@ fn convert_within_budget( let elapsed = started.elapsed(); converted.and_then(|converted| within_budget(converted, (sql, elapsed), options)) } - -/// `converted`, or a refusal when converting it outran the time or size budget. -fn within_budget( - converted: Converted, - (source, elapsed): (&str, Duration), - options: &ConversionOptions, -) -> Result { - let budget = options.budget; - let emitted = converted.sql.len() as u64; - let allowed = budget - .minimum_bytes - .max((source.len() as u64).saturating_mul(budget.size_ratio)); - let reason = if elapsed > Duration::from_millis(budget.milliseconds) { - format!( - "converting the statement took {} ms, beyond the budget of {} ms", - elapsed.as_millis(), - budget.milliseconds - ) - } else if emitted > allowed { - format!( - "the DuckDB SQL is {emitted} bytes for {} bytes of source, beyond the budget of \ - {allowed} bytes ({} times the source, at least {})", - source.len(), - budget.size_ratio, - budget.minimum_bytes - ) - } else { - return Ok(converted); - }; - Err(Refusal { - construct: None, - location: None, - category: RefusalCategory::ConversionBudget, - reason, - }) -} diff --git a/crates/sqlquack/src/analysis/main/mod.rs b/crates/sqlquack/src/analysis/main/mod.rs index 3fb42d2d6..e76c414b5 100644 --- a/crates/sqlquack/src/analysis/main/mod.rs +++ b/crates/sqlquack/src/analysis/main/mod.rs @@ -7,12 +7,8 @@ pub mod check_equivalence_cases; pub mod check_source; pub mod collect_all_refusals; pub mod confirm_assumption; -pub mod confirm_utc_session; -pub mod conversion_options; pub mod override_macros; -pub mod parse_session; pub mod python_functions; pub mod refuse_value_divergences; pub mod refused_construct; -pub mod session_conversion_options; pub mod with_overrides; diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index cd74db1ae..4ca4f0d69 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -24,13 +24,13 @@ use crate::analysis::main::analyze_function::analyze_function; use crate::analysis::main::check_equivalence_cases::check_equivalence_cases; use crate::analysis::main::collect_all_refusals::collect_all_refusals; use crate::analysis::main::confirm_assumption::confirm_assumption; -use crate::analysis::main::confirm_utc_session::confirm_utc_session; -use crate::analysis::main::conversion_options::conversion_options; use crate::analysis::main::refused_construct::refused_construct; use crate::analysis::models::PipelineConverter; use crate::analysis::tests::test_types::{ AnalysisSummary, CollectedSummary, DivergenceModeOutcome, }; +use sqlquack_rules::session::main::confirm_utc_session::confirm_utc_session; +use sqlquack_rules::session::main::conversion_options::conversion_options; fn shop_schema() -> Schema { let mut schema = shop_relations(); diff --git a/crates/sqlquack/src/census/tests/helpers.rs b/crates/sqlquack/src/census/tests/helpers.rs index 78f393724..f71ef464e 100644 --- a/crates/sqlquack/src/census/tests/helpers.rs +++ b/crates/sqlquack/src/census/tests/helpers.rs @@ -8,7 +8,6 @@ use sqlquack_rules::conversion::models::{AllowedDivergence, ConversionOptions}; use sqlquack_semantics::types::models::{ColumnSpec, Schema}; use sqlquack_snowflake::typing::models::SnowflakeType; -use crate::analysis::main::conversion_options::conversion_options; use crate::census::_helpers::coverage::{CoverageInput, greedy_order}; use crate::census::_helpers::timed::run_census_timed; use crate::census::main::render_census_text::render_census_text; @@ -17,6 +16,7 @@ use crate::census::models::{CensusOptions, CensusReport}; use crate::census::tests::test_types::{ BlockerSummary, CensusSummary, PlanSummary, ShopFile, UpstreamSummary, }; +use sqlquack_rules::session::main::conversion_options::conversion_options; pub(crate) const SHOP: &[ShopFile] = &[ ShopFile { diff --git a/crates/sqlquack/src/cli/_helpers/evidence.rs b/crates/sqlquack/src/cli/_helpers/evidence.rs index f90c3e8e3..51681546e 100644 --- a/crates/sqlquack/src/cli/_helpers/evidence.rs +++ b/crates/sqlquack/src/cli/_helpers/evidence.rs @@ -10,11 +10,11 @@ use sqlquack_policy::policy::models::Policy; use sqlquack_rules::conversion::models::ConversionOptions; use crate::analysis::main::confirm_assumption::confirm_assumption; -use crate::analysis::main::confirm_utc_session::confirm_utc_session; -use crate::analysis::main::conversion_options::conversion_options; use crate::analysis::main::refuse_value_divergences::refuse_value_divergences; use crate::cli::models::{EvidencePaths, PolicyArguments}; use sqlquack_rules::conversion::constants::{DECLARED_OFFSET, PROJECT_UDF}; +use sqlquack_rules::session::main::confirm_utc_session::confirm_utc_session; +use sqlquack_rules::session::main::conversion_options::conversion_options; pub(crate) struct Evidence { pub(crate) policy: Policy, diff --git a/crates/sqlquack/src/cli/_helpers/function_command.rs b/crates/sqlquack/src/cli/_helpers/function_command.rs index b6376b8ca..3b1a57c28 100644 --- a/crates/sqlquack/src/cli/_helpers/function_command.rs +++ b/crates/sqlquack/src/cli/_helpers/function_command.rs @@ -17,13 +17,13 @@ use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; use sqlquack_rules::conversion::models::ConvertedFunction; use crate::analysis::main::analyze_function::analyze_function; -use crate::analysis::main::parse_session::parse_session; -use crate::analysis::main::session_conversion_options::session_conversion_options; use crate::census::main::resolve_references::resolve_references; use crate::cli::_helpers::evidence::caller_choices; use crate::cli::_helpers::evidence::chosen_policy; use crate::cli::_helpers::schema::load_schema; use crate::cli::models::FunctionArguments; +use sqlquack_rules::session::main::parse_session::parse_session; +use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; /// The path naming standard input. const STDIN_PATH: &str = "-"; diff --git a/crates/sqlquack/src/cli/_helpers/review_command.rs b/crates/sqlquack/src/cli/_helpers/review_command.rs index ea9f2b718..d3b2520df 100644 --- a/crates/sqlquack/src/cli/_helpers/review_command.rs +++ b/crates/sqlquack/src/cli/_helpers/review_command.rs @@ -3,12 +3,12 @@ use std::io::Write; use std::process::ExitCode; -use crate::analysis::main::conversion_options::conversion_options; use crate::analysis::models::PipelineConverter; use crate::cli::_helpers::verify_command::run_verification; use crate::cli::models::ReviewArguments; use crate::review::main::export_review::export_review; use crate::review::models::ReviewFile; +use sqlquack_rules::session::main::conversion_options::conversion_options; pub(crate) fn review_command(arguments: &ReviewArguments, stderr: &mut dyn Write) -> ExitCode { let _ = writeln!( diff --git a/crates/sqlquack/src/cli/_helpers/schema.rs b/crates/sqlquack/src/cli/_helpers/schema.rs index 1259947f7..7f733c98e 100644 --- a/crates/sqlquack/src/cli/_helpers/schema.rs +++ b/crates/sqlquack/src/cli/_helpers/schema.rs @@ -1,207 +1,17 @@ //! Reading a schema file: typed relations, and the signatures of project functions. -use std::collections::BTreeMap; use std::path::Path; -use serde::Deserialize; -use sqlquack_core::provenance::models::ProvenanceLog; -use sqlquack_semantics::types::models::{ - ColumnSpec, FunctionLanguage, FunctionReturns, FunctionSpec, FunctionTraits, NullInput, Schema, - Volatility, -}; -use sqlquack_snowflake::typing::main::cast_target::cast_target; -use sqlquack_snowflake::typing::models::SnowflakeType; - -use crate::analysis::constants::DECLARABLE_OFFSETS; - -/// Columns and parameters as the schema file writes them. -type RawColumns = Vec; - -/// A name and Snowflake type pair, or an object that may also mark a VARIANT's typed leaves. -#[derive(Deserialize)] -#[serde(untagged)] -enum RawColumn { - Pair(String, String), - Described(RawDescribedColumn), -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawDescribedColumn { - name: String, - #[serde(rename = "type")] - type_name: String, - #[serde(default)] - typed_leaves: bool, - /// The offset every value of a TIMESTAMP_TZ source column holds: `+00:00` or `UTC`. - #[serde(default)] - offset: Option, -} - -/// The schema file: relations and functions, or relations alone as the top-level map. -#[derive(Deserialize)] -#[serde(untagged)] -enum RawSchema { - Sections(RawSections), - Relations(BTreeMap), -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawSections { - #[serde(default)] - relations: BTreeMap, - #[serde(default)] - functions: BTreeMap, -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawFunction { - language: RawLanguage, - #[serde(default)] - arguments: RawColumns, - returns: RawReturns, - #[serde(default)] - volatility: Volatility, - #[serde(default)] - null_input: NullInput, - #[serde(default)] - vectorized: bool, -} - -#[derive(Clone, Copy, Deserialize)] -#[serde(rename_all = "lowercase")] -enum RawLanguage { - Sql, - Python, -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum RawReturns { - Scalar(String), - Table { table: RawColumns }, -} +use sqlquack_semantics::types::errors::SchemaError; +use sqlquack_semantics::types::main::parse_schema::parse_schema; +use sqlquack_semantics::types::models::Schema; /// The schema in `path`; a relation or function using a type SQLQuack does not know is left out. pub(crate) fn load_schema(path: &Path) -> Result { let text = std::fs::read_to_string(path) .map_err(|error| format!("cannot read {}: {error}", path.display()))?; - let raw: RawSchema = serde_json::from_str(&text) - .map_err(|error| format!("{} is not a schema file: {error}", path.display()))?; - let (relations, functions) = match raw { - RawSchema::Sections(sections) => (sections.relations, sections.functions), - RawSchema::Relations(relations) => (relations, BTreeMap::new()), - }; - let log = ProvenanceLog::default(); - let mut schema = Schema::default(); - for (relation, columns) in relations { - if let Some(typed) = typed_columns(columns, Some(&relation), &log)? { - schema.relations.insert(relation, typed); - } - } - for (name, function) in functions { - if let Some(typed) = typed_function(function, &log)? { - schema.functions.insert(name, typed); - } - } - Ok(schema) -} - -fn typed_function( - function: RawFunction, - log: &ProvenanceLog, -) -> Result, String> { - let returns = match function.returns { - RawReturns::Scalar(type_name) => match cast_target(&type_name, log) { - Ok(target) => FunctionReturns::Scalar(target.data_type), - Err(_) => return Ok(None), - }, - RawReturns::Table { table } => match typed_columns(table, None, log)? { - Some(columns) => FunctionReturns::Table(columns), - None => return Ok(None), - }, - }; - let Some(arguments) = typed_columns(function.arguments, None, log)? else { - return Ok(None); - }; - Ok(Some(FunctionSpec { - language: match function.language { - RawLanguage::Sql => FunctionLanguage::Sql, - RawLanguage::Python => FunctionLanguage::Python, - }, - arguments, - returns, - traits: FunctionTraits { - volatility: function.volatility, - null_input: function.null_input, - vectorized: function.vectorized, - }, - })) -} - -/// The columns of `relation`, or of a function when None; None when a type is unknown. -fn typed_columns( - columns: RawColumns, - relation: Option<&str>, - log: &ProvenanceLog, -) -> Result>, String> { - let mut typed: Vec = Vec::with_capacity(columns.len()); - for column in columns { - let (name, type_name, typed_leaves, offset) = match column { - RawColumn::Pair(name, type_name) => (name, type_name, false, None), - RawColumn::Described(column) => ( - column.name, - column.type_name, - column.typed_leaves, - column.offset, - ), - }; - let Ok(target) = cast_target(&type_name, log) else { - return Ok(None); - }; - let declared = match offset { - None => Vec::new(), - Some(offset) => vec![declared_offset( - (relation, &name), - target.data_type, - &offset, - )?], - }; - typed.push(ColumnSpec { - typed_leaves, - utc_offset: !declared.is_empty(), - declared_offsets: declared, - ..ColumnSpec::new(name, target.data_type) - }); - } - Ok(Some(typed)) -} - -/// The label of a relation column declared to hold the UTC offset; any other declaration fails. -fn declared_offset( - (relation, name): (Option<&str>, &str), - data_type: SnowflakeType, - offset: &str, -) -> Result { - let Some(relation) = relation else { - return Err(format!( - "{name}: an offset may be declared only for a relation's column" - )); - }; - if !matches!(data_type, SnowflakeType::TimestampTz { .. }) { - return Err(format!( - "{relation}.{name}: an offset may be declared only for a TIMESTAMP_TZ column, not {data_type}" - )); - } - if !DECLARABLE_OFFSETS - .iter() - .any(|declarable| declarable.eq_ignore_ascii_case(offset)) - { - return Err(format!( - "{relation}.{name}: the declared offset `{offset}` is not supported; only +00:00 (UTC) is" - )); - } - Ok(format!("{relation}.{name}")) + parse_schema(&text).map_err(|error| match error { + SchemaError::Shape(error) => format!("{} is not a schema file: {error}", path.display()), + SchemaError::Declaration(reason) => reason, + }) } diff --git a/crates/sqlquack/src/cli/_helpers/verify_command.rs b/crates/sqlquack/src/cli/_helpers/verify_command.rs index 9ed8da25d..b309bc76a 100644 --- a/crates/sqlquack/src/cli/_helpers/verify_command.rs +++ b/crates/sqlquack/src/cli/_helpers/verify_command.rs @@ -35,12 +35,12 @@ use crate::analysis::main::builtin_claims::builtin_claims; use crate::analysis::main::builtin_failure_modes::builtin_failure_modes; use crate::analysis::main::check_equivalence_cases::check_equivalence_cases; use crate::analysis::main::confirm_assumption::confirm_assumption; -use crate::analysis::main::conversion_options::conversion_options; use crate::analysis::models::PipelineConverter; use crate::cli::_helpers::evidence::{Evidence, load_evidence}; use crate::cli::models::{ CompareArguments, Comparison, CoverageArguments, EvidencePaths, VerifyArguments, }; +use sqlquack_rules::session::main::conversion_options::conversion_options; pub(crate) fn verify_command( arguments: &VerifyArguments, diff --git a/crates/sqlquack/src/cli/main/convert_report.rs b/crates/sqlquack/src/cli/main/convert_report.rs index 62f939ac7..177b4a6cc 100644 --- a/crates/sqlquack/src/cli/main/convert_report.rs +++ b/crates/sqlquack/src/cli/main/convert_report.rs @@ -4,12 +4,12 @@ use sqlquack_core::parsing::models::SourceDialect; use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; use crate::analysis::main::analyze::analyze; -use crate::analysis::main::parse_session::parse_session; -use crate::analysis::main::session_conversion_options::session_conversion_options; use crate::cli::_helpers::convert_command::{analysis_report, chosen_options, refused_session}; use crate::cli::_helpers::evidence::chosen_policy; use crate::cli::_helpers::schema::load_schema; use crate::cli::models::{ConvertArguments, ConvertReport}; +use sqlquack_rules::session::main::parse_session::parse_session; +use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; /// The report `convert` prints for `sql` under `arguments`, or why it cannot be made (exit 2). pub fn convert_report(arguments: &ConvertArguments, sql: &str) -> Result { diff --git a/website/.gitignore b/website/.gitignore index 30e70a8cc..0091ba03d 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -3,3 +3,5 @@ dist/ .astro/ public/playground/engine/ wasm/target/ +public/playground/duckdb/ +public/playground/sqlquack.toml diff --git a/website/README.md b/website/README.md index 2ba0eeb86..f55a81cac 100644 --- a/website/README.md +++ b/website/README.md @@ -14,16 +14,23 @@ structure, styling and components with the SQLClean and SQLBuild websites. - `src/components/`: the page layout, header and page title, replacing Starlight's defaults. - `public/fonts/`: self-hosted fonts, each with its licence. - `public/playground/`: the playground page. `wasm/` is its engine, a thin wasm-bindgen crate over - the existing SQLQuack crates (parse, Snowflake checks, name checks, conversion under the built-in - policy), kept out of the main Cargo workspace as `fuzz/` is. `scripts/build_engine.sh` builds it - into `public/playground/engine/`, which is not committed. Translated SQL runs in DuckDB-WASM, - loaded from jsDelivr on first use. + the existing SQLQuack crates (parse, Snowflake checks, name checks, conversion under a policy and + session), kept out of the main Cargo workspace as `fuzz/` is. `scripts/build_engine.sh` builds it + into `public/playground/engine/`, which is not committed. The engine runs in a Web Worker. + Translated SQL runs in DuckDB-WASM, served from the site: `scripts/prepare_playground.mjs` copies + it from `node_modules` into `public/playground/duckdb/` on every `npm run build` and `npm run dev`. + +DuckDB-WASM is pinned to `1.33.1-dev114.0`, a prerelease from npm's `next` tag and the first build +that bundles DuckDB 2.0. Swap it for a stable 2.x release once one ships: change the version in +`package.json`, run `npm install`, and check that stage 4 of the playground reports the expected +DuckDB version. The logo uses the SQLQuack icon from `sqlstack-io/sqlstack` (`assets/icons/sqlquack.png`); the footer's SQLStack symbols are unchanged copies from the same repository. -The Website workflow builds the site and checks its links when run by hand. It does not deploy, -and nothing publishes the site yet. +The Website workflow builds and tests the engine, builds the site and checks its links, on pull +requests that touch them and when run by hand. It never runs on push to main and does not deploy; +nothing publishes the site yet. Requires Node 22.12 or newer. @@ -37,5 +44,5 @@ npm run dev # live preview npm run build # static site in dist/, including the search index npm run preview # serve dist/ python3 scripts/check_links.py dist -npm test # Playwright: branding and navigation +npm run test:engine # the engine and its spans, after build_engine.sh ``` diff --git a/website/package-lock.json b/website/package-lock.json index 426881ebc..0b26064e8 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -7,11 +7,15 @@ "name": "sqlquack-website", "dependencies": { "@astrojs/starlight": "^0.42.4", + "@duckdb/duckdb-wasm": "1.33.1-dev114.0", "@lucide/astro": "^1.48.0", "astro": "^7.3.5", "sharp": "^0.35.4", "starlight-llms-txt": "^0.12.0" }, + "devDependencies": { + "esbuild": "0.28.2" + }, "engines": { "node": ">=22.12.0" } @@ -593,6 +597,16 @@ "node": ">=14" } }, + "node_modules/@duckdb/duckdb-wasm": { + "version": "1.33.1-dev114.0", + "resolved": "https://registry.npmjs.org/@duckdb/duckdb-wasm/-/duckdb-wasm-1.33.1-dev114.0.tgz", + "integrity": "sha512-0sk+bv9iY7GuDQ0n0BlXBO6FucZ8PPVzGswOJh6kXoGuPe0SJRc49oyn+PNcz2JNE2XmD+Wi60fRUjTloHn6bw==", + "license": "MIT", + "dependencies": { + "apache-arrow": "^17.0.0", + "qs": "^6.14.1" + } + }, "node_modules/@emnapi/core": { "version": "1.11.3", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", @@ -2079,6 +2093,12 @@ "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", "license": "MIT" }, + "node_modules/@swc/helpers": { + "version": "0.5.24", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.24.tgz", + "integrity": "sha512-6tcXo4kA8HgGO7AVmawB57RYi/piBPYVHVcO6dXjX+SZw8GoHmi262sp4v2Ha2kNP9Cx84qei27yT+fjS1lxgg==", + "license": "Apache-2.0" + }, "node_modules/@tybys/wasm-util": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", @@ -2095,6 +2115,18 @@ "integrity": "sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w==", "license": "MIT" }, + "node_modules/@types/command-line-args": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", + "license": "MIT" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", + "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", + "license": "MIT" + }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -2209,6 +2241,21 @@ "am-i-vibing": "dist/cli.mjs" } }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -2234,6 +2281,41 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/apache-arrow": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-17.0.0.tgz", + "integrity": "sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.11", + "@types/command-line-args": "^5.2.3", + "@types/command-line-usage": "^5.0.4", + "@types/node": "^20.13.0", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^24.3.25", + "json-bignum": "^0.0.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.cjs" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -2255,6 +2337,15 @@ "node": ">= 0.4" } }, + "node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/astro": { "version": "7.3.5", "resolved": "https://registry.npmjs.org/astro/-/astro-7.3.5.tgz", @@ -2414,6 +2505,35 @@ "node": ">=8" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/ccount": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", @@ -2424,6 +2544,37 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", @@ -2503,6 +2654,24 @@ "node": ">=6" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -2513,6 +2682,54 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "license": "MIT", + "dependencies": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-usage": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz", + "integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^4.1.1", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/command-line-usage/node_modules/array-back": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", + "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/command-line-usage/node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/commander": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", @@ -2838,6 +3055,20 @@ "node": ">=4" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/entities": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", @@ -2850,12 +3081,42 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -2995,6 +3256,24 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "license": "MIT", + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/flatbuffers": { + "version": "24.12.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz", + "integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==", + "license": "Apache-2.0" + }, "node_modules/flattie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/flattie/-/flattie-1.1.1.tgz", @@ -3039,6 +3318,52 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-tsconfig": { "version": "5.0.0-beta.4", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.4.tgz", @@ -3060,6 +3385,18 @@ "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", "license": "ISC" }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/h3": { "version": "1.15.11", "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.11.tgz", @@ -3077,6 +3414,39 @@ "uncrypto": "^0.1.3" } }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/hast-util-embedded": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz", @@ -3540,6 +3910,14 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/jsonc-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", @@ -3804,6 +4182,12 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -3853,6 +4237,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mdast-util-directive": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mdast-util-directive/-/mdast-util-directive-3.1.0.tgz", @@ -4785,6 +5178,18 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/obug": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz", @@ -5054,6 +5459,22 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/qs": { + "version": "6.17.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.17.1.tgz", + "integrity": "sha512-jd3COkaKVgGUW+ZHyEHCG9bH7RoaaiWuMgzsIBjUMVjLAaWGWpoUAVwljVvBHxRu/2qEHnadLGZrkutYUBPQJA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/radix3": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/radix3/-/radix3-1.1.2.tgz", @@ -5386,6 +5807,78 @@ "node": ">=20" } }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -5480,6 +5973,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/svgo": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/svgo/-/svgo-4.1.0.tgz", @@ -5505,6 +6010,28 @@ "url": "https://opencollective.com/svgo" } }, + "node_modules/table-layout": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", + "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "wordwrapjs": "^5.1.0" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/array-back": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", + "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/tiny-inflate": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", @@ -5591,8 +6118,16 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "license": "0BSD" + }, + "node_modules/typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", + "license": "MIT", + "engines": { + "node": ">=8" + } }, "node_modules/ufo": { "version": "1.6.4", @@ -6028,6 +6563,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/wordwrapjs": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", + "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/xxhash-wasm": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", diff --git a/website/package.json b/website/package.json index 19801c7e0..6527de0a3 100644 --- a/website/package.json +++ b/website/package.json @@ -6,15 +6,20 @@ "node": ">=22.12.0" }, "scripts": { - "dev": "astro dev", - "build": "astro build", - "preview": "astro preview" + "dev": "node scripts/prepare_playground.mjs && astro dev", + "build": "node scripts/prepare_playground.mjs && astro build", + "preview": "astro preview", + "test:engine": "node --test tests/playground.test.mjs" }, "dependencies": { "@astrojs/starlight": "^0.42.4", + "@duckdb/duckdb-wasm": "1.33.1-dev114.0", "@lucide/astro": "^1.48.0", "astro": "^7.3.5", "sharp": "^0.35.4", "starlight-llms-txt": "^0.12.0" + }, + "devDependencies": { + "esbuild": "0.28.2" } } diff --git a/website/playwright.config.mjs b/website/playwright.config.mjs deleted file mode 100644 index 8aafb91d4..000000000 --- a/website/playwright.config.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import { defineConfig } from '@playwright/test'; - -export default defineConfig({ - testDir: './tests', - use: { - baseURL: process.env.SITE_URL || 'http://127.0.0.1:4174', - deviceScaleFactor: 2, - launchOptions: process.env.CHROME_PATH ? { executablePath: process.env.CHROME_PATH } : {}, - }, - webServer: process.env.SITE_URL ? undefined : { - command: 'npm run preview -- --host 127.0.0.1 --port 4174 --ignore-lock', - url: 'http://127.0.0.1:4174', - }, -}); diff --git a/website/public/playground/index.html b/website/public/playground/index.html index 8d16c56fb..c23348ffe 100644 --- a/website/public/playground/index.html +++ b/website/public/playground/index.html @@ -89,6 +89,14 @@

Playground

DuckDB SQL run after the schema's tables are created, before the translated query.

+
+ Policy and session + +

A policy file, as for --policy.

+ +

Snowflake session parameters, one NAME=VALUE per line, as for --session.

+

+
@@ -100,12 +108,13 @@

Playground

-
3

Translate to DuckDB

built-in policy…
+
3

Translate to DuckDB

…
4

Run in DuckDB-WASM

…
+

Runs DuckDB-WASM in your browser, a preview build. SQLQuack's checks rest on native DuckDB 2.0; this run is a demonstration, not that check.

diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index cb019a814..359f0e3ba 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -1,8 +1,29 @@ -// The SQLQuack playground: each keystroke runs the engine (SQLQuack compiled to WebAssembly) and -// shows its stages; translated SQL runs in DuckDB-WASM, loaded from jsDelivr on first use. -import init, { analyze, fixture_tables } from './engine/sqlquack_wasm.js'; +// The SQLQuack playground: each edit runs the engine (SQLQuack compiled to WebAssembly, in a +// worker) and shows its stages; translated SQL runs in DuckDB-WASM. -const DUCKDB_WASM = 'https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.32.0/+esm'; +import { escape, rendered, underlined } from './spans.js'; + +// The engine's worker; `engine(op, fields)` resolves with its reply. +const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' }); +const pending = new Map(); +let nextId = 0; +const engineReady = new Promise((resolve) => { + worker.addEventListener('message', ({ data }) => { + if (data.ready) return resolve(); + pending.get(data.id)?.(data); + pending.delete(data.id); + }); +}); +function engine(op, fields) { + const id = ++nextId; + return new Promise((resolve) => { + pending.set(id, resolve); + worker.postMessage({ id, op, ...fields }); + }); +} + +// DuckDB-WASM, served from this site (scripts/prepare_playground.mjs copies it in at build time). +const DUCKDB_WASM = new URL('./duckdb/', import.meta.url); const SCHEMA = `{ "ORDERS": [ @@ -51,42 +72,15 @@ const $ = (selector) => document.querySelector(selector); const sqlBox = $('#pg-sql'); const schemaBox = $('#pg-schema'); const rowsBox = $('#pg-rows'); +const policyBox = $('#pg-policy'); +const sessionBox = $('#pg-session'); +const strictBox = $('#pg-strict'); +const utcBox = $('#pg-utc'); +const settings = () => + JSON.stringify({ policy: policyBox.value, session: sessionBox.value, strict: strictBox.checked, utc_session: utcBox.checked }); const marks = $('#pg-marks'); const tip = $('#pg-tip'); -const escape = (text) => text.replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c]); - -// Spans count Unicode characters; JavaScript strings count UTF-16 units, so index by code point. -function position(chars, offset) { - let line = 1; - let column = 1; - for (let i = 0; i < offset && i < chars.length; i++) { - if (chars[i] === '\n') { - line += 1; - column = 1; - } else column += 1; - } - return { line, column }; -} - -// One diagnostic as the command line renders it, rustc style. -function rendered(diagnostic, chars) { - const head = `error[${escape(diagnostic.code)}]: ${escape(diagnostic.message)}`; - const span = diagnostic.span; - if (!span) return head + (diagnostic.label ? `\n = ${escape(diagnostic.label)}` : ''); - const { line, column } = span.line ? span : position(chars, span.start); - const lines = chars.join('').split('\n'); - const text = lines[line - 1] ?? ''; - const endOnLine = span.end_line && span.end_line !== line ? text.length + 1 : column + Math.max(1, span.end - span.start); - const width = Math.max(1, endOnLine - column); - const gutter = ' '.repeat(String(line).length); - let out = `${head}\n${gutter}--> query.sql:${line}:${column}\n${gutter} |\n${line} | ${escape(text)}\n${gutter} | ${' '.repeat(column - 1)}${'^'.repeat(width)}`; - if (diagnostic.label) out += ` ${escape(diagnostic.label)}`; - for (const note of diagnostic.notes ?? []) out += `\n${gutter} = note: ${escape(note)}`; - if (diagnostic.help) out += `\n${gutter} = help: ${escape(diagnostic.help)}`; - return out; -} - function setStage(id, status, body, sub) { const stage = $(`#stage-${id}`); const pill = stage.querySelector('.pg-status'); @@ -100,41 +94,28 @@ const terminal = (html) => `
${html}
`; // Underlines every span in the editor; overlapping spans keep the first. function underline(chars, items) { - const sorted = items.filter((item) => item.span).sort((a, b) => a.span.start - b.span.start); - let html = ''; - let at = 0; - sorted.forEach((item, index) => { - const start = Math.min(item.span.start, chars.length); - let end = Math.min(Math.max(item.span.end, start + 1), chars.length); - if (start < at) return; - if (end === start) { - // A span at the very end, such as a missing token: underline the last character. - if (start === 0) return; - html += escape(chars.slice(at, start - 1).join('')); - html += `${escape(chars[start - 1])}`; - at = start; - return; - } - html += escape(chars.slice(at, start).join('')); - html += `${escape(chars.slice(start, end).join(''))}`; - at = end; - }); - html += escape(chars.slice(at).join('')) + '\n'; + const { html, items: kept } = underlined(chars, items); marks.innerHTML = html; - marks.items = sorted; + marks.items = kept; marks.scrollTop = sqlBox.scrollTop; } let latest = null; +let current = 0; -function run() { +async function run() { + const ticket = ++current; const sql = sqlBox.value; + const reply = await engine('analyze', { sql, schema: schemaBox.value, settings: settings() }); + // A newer edit has already been sent; its reply will render instead. + if (ticket !== current) return; const chars = Array.from(sql); - const report = JSON.parse(analyze(sql, schemaBox.value)); + const report = JSON.parse(reply.result); latest = null; - if (report.schema_error) { + if (report.schema_error || report.settings_error) { underline(chars, []); - setStage('parse', 'skipped', terminal(`The schema is not valid: ${escape(report.schema_error)}`)); + const what = report.schema_error ? 'The schema is not valid' : 'The policy or session is not valid'; + setStage('parse', 'skipped', terminal(`${what}: ${escape(report.schema_error ?? report.settings_error)}`)); for (const id of ['checks', 'translate', 'run']) setStage(id, 'skipped'); return; } @@ -178,14 +159,14 @@ function run() { body = (translate.statements ?? []).map((s) => terminal(escape(s.sql))).join(''); body += (translate.diagnostics ?? []).map((d) => terminal(rendered(d, chars))).join(''); } - setStage('translate', translate.status, body); + setStage('translate', translate.status, body, translate.profile ? `policy: ${translate.profile}` : ''); underline(chars, spans); const statements = translate.statements ?? []; if (translate.status === 'pass' && statements.length === 1) { latest = statements[0].sql; if (duckdb) execute(); - else setStage('run', 'ready', '', 'loads DuckDB from jsDelivr'); + else setStage('run', 'ready', '', 'loads DuckDB-WASM (about 8 MB compressed)'); } else if (translate.status === 'pass') { setStage('run', 'skipped', terminal('Runs one statement at a time.')); } else setStage('run', 'skipped', ''); @@ -196,12 +177,10 @@ let duckdb = null; let running = 0; async function loadDuckDB() { - const module = await import(DUCKDB_WASM); - const bundle = await module.selectBundle(module.getJsDelivrBundles()); - const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: 'text/javascript' })); - const db = new module.AsyncDuckDB(new module.VoidLogger(), new Worker(workerUrl)); - await db.instantiate(bundle.mainModule, bundle.pthreadWorker); - URL.revokeObjectURL(workerUrl); + const module = await import(new URL('duckdb-browser.mjs', DUCKDB_WASM).href); + const worker = new Worker(new URL('duckdb-browser-base.worker.js', DUCKDB_WASM)); + const db = new module.AsyncDuckDB(new module.VoidLogger(), worker); + await db.instantiate(new URL('duckdb-base.wasm', DUCKDB_WASM).href); const version = await (await db.connect()).query('SELECT version() AS v'); return { db, version: String(version.toArray()[0].v) }; } @@ -215,7 +194,7 @@ async function execute() { if (!duckdb) duckdb = await loadDuckDB(); const conn = await duckdb.db.connect(); try { - const tables = JSON.parse(fixture_tables(schemaBox.value)); + const tables = JSON.parse((await engine('fixture', { schema: schemaBox.value })).result); if (tables.error) throw new Error(`schema: ${tables.error}`); for (const table of tables.tables) await conn.query(table.replace(/^CREATE TABLE/, 'CREATE OR REPLACE TABLE')); if (rowsBox.value.trim()) await conn.query(rowsBox.value); @@ -250,6 +229,11 @@ document.addEventListener('click', (event) => { }); sqlBox.addEventListener('input', schedule); schemaBox.addEventListener('input', schedule); +for (const box of [policyBox, sessionBox, strictBox, utcBox]) box.addEventListener('input', schedule); +$('#pg-load-policy').addEventListener('click', async () => { + policyBox.value = await (await fetch(new URL('./sqlquack.toml', import.meta.url))).text(); + schedule(); +}); rowsBox.addEventListener('input', () => latest && duckdb && schedule()); sqlBox.addEventListener('scroll', () => (marks.scrollTop = sqlBox.scrollTop)); $('#pg-example').addEventListener('change', (event) => { @@ -281,6 +265,6 @@ sqlBox.value = params.get('sql') ?? EXAMPLES[params.get('example')] ?? EXAMPLES. schemaBox.value = SCHEMA; rowsBox.value = ROWS; setStage('parse', 'loading', '', 'loading the engine'); -await init(); -run(); +await engineReady; +await run(); if (params.has('run')) execute(); diff --git a/website/public/playground/spans.js b/website/public/playground/spans.js new file mode 100644 index 000000000..46f240576 --- /dev/null +++ b/website/public/playground/spans.js @@ -0,0 +1,62 @@ +// Diagnostic spans in the playground. The engine counts Unicode characters (code points), and +// JavaScript strings count UTF-16 units, so every span is applied to `Array.from(sql)`. + +export const escape = (text) => text.replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c]); + +// Spans count Unicode characters; JavaScript strings count UTF-16 units, so index by code point. +export function position(chars, offset) { + let line = 1; + let column = 1; + for (let i = 0; i < offset && i < chars.length; i++) { + if (chars[i] === '\n') { + line += 1; + column = 1; + } else column += 1; + } + return { line, column }; +} + +// One diagnostic as the command line renders it, rustc style. +export function rendered(diagnostic, chars) { + const head = `error[${escape(diagnostic.code)}]: ${escape(diagnostic.message)}`; + const span = diagnostic.span; + if (!span) return head + (diagnostic.label ? `\n = ${escape(diagnostic.label)}` : ''); + const { line, column } = span.line ? span : position(chars, span.start); + const lines = chars.join('').split('\n'); + const text = lines[line - 1] ?? ''; + const endOnLine = span.end_line && span.end_line !== line ? text.length + 1 : column + Math.max(1, span.end - span.start); + const width = Math.max(1, endOnLine - column); + const gutter = ' '.repeat(String(line).length); + let out = `${head}\n${gutter}--> query.sql:${line}:${column}\n${gutter} |\n${line} | ${escape(text)}\n${gutter} | ${' '.repeat(column - 1)}${'^'.repeat(width)}`; + if (diagnostic.label) out += ` ${escape(diagnostic.label)}`; + for (const note of diagnostic.notes ?? []) out += `\n${gutter} = note: ${escape(note)}`; + if (diagnostic.help) out += `\n${gutter} = help: ${escape(diagnostic.help)}`; + return out; +} + +// The editor's underline layer as HTML: every span marked; overlapping spans keep the first. +// `chars` is the SQL split into Unicode characters, the unit the engine's spans count. +export function underlined(chars, items) { + const sorted = items.filter((item) => item.span).sort((a, b) => a.span.start - b.span.start); + let html = ''; + let at = 0; + sorted.forEach((item, index) => { + const start = Math.min(item.span.start, chars.length); + let end = Math.min(Math.max(item.span.end, start + 1), chars.length); + if (start < at) return; + if (end === start) { + // A span at the very end, such as a missing token: underline the last character. + if (start === 0) return; + html += escape(chars.slice(at, start - 1).join('')); + html += `${escape(chars[start - 1])}`; + at = start; + return; + } + html += escape(chars.slice(at, start).join('')); + html += `${escape(chars.slice(start, end).join(''))}`; + at = end; + }); + html += escape(chars.slice(at).join('')) + '\n'; + return { html, items: sorted }; +} + diff --git a/website/public/playground/worker.js b/website/public/playground/worker.js new file mode 100644 index 000000000..24b3f0e36 --- /dev/null +++ b/website/public/playground/worker.js @@ -0,0 +1,14 @@ +// Runs the SQLQuack engine off the page's thread, so typing never waits on a long statement. +// Each request carries an id; the reply echoes it, so the page can drop replies it no longer needs. +import init, { analyze, fixture_tables } from './engine/sqlquack_wasm.js'; + +const ready = init(); + +self.onmessage = async ({ data }) => { + await ready; + const started = performance.now(); + const result = data.op === 'fixture' ? fixture_tables(data.schema) : analyze(data.sql, data.schema, data.settings); + self.postMessage({ id: data.id, result, elapsed: performance.now() - started }); +}; + +ready.then(() => self.postMessage({ ready: true })); diff --git a/website/public/styles.css b/website/public/styles.css index dd9255402..9df4926b2 100644 --- a/website/public/styles.css +++ b/website/public/styles.css @@ -1216,6 +1216,26 @@ footer a { font-size: 12px; } +.pg-note label { + margin-right: 16px; + cursor: pointer; +} + +.pg-small.pg-short { + min-height: 56px; + margin-top: 12px; +} + +.pg-link { + padding: 0; + border: 0; + background: none; + color: var(--primary); + font: inherit; + text-decoration: underline; + cursor: pointer; +} + .pg-stages { display: grid; gap: 12px; diff --git a/website/scripts/prepare_playground.mjs b/website/scripts/prepare_playground.mjs new file mode 100644 index 000000000..884214453 --- /dev/null +++ b/website/scripts/prepare_playground.mjs @@ -0,0 +1,27 @@ +// Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`: +// DuckDB-WASM from node_modules into public/playground/duckdb/, so the playground serves it from +// the site rather than a CDN (its browser module bundled with its Apache Arrow import), and the +// repository's policy file, which the page offers to load. Run from website/. +import { build } from 'esbuild'; +import { copyFileSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; + +const root = 'node_modules/@duckdb/duckdb-wasm'; +const { version } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); +const out = 'public/playground/duckdb'; + +rmSync(out, { recursive: true, force: true }); +mkdirSync(out, { recursive: true }); +await build({ + entryPoints: [join(root, 'dist/duckdb-browser.mjs')], + bundle: true, + format: 'esm', + minify: true, + outfile: join(out, 'duckdb-browser.mjs'), + logLevel: 'warning', +}); +for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) { + copyFileSync(join(root, 'dist', file), join(out, file)); +} +console.log(`duckdb-wasm ${version} vendored into ${out}`); +copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml'); diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs new file mode 100644 index 000000000..dd03b4156 --- /dev/null +++ b/website/tests/playground.test.mjs @@ -0,0 +1,83 @@ +// The playground engine and its span handling, under Node's test runner: `npm run test:engine`. +// Needs the engine built into public/playground/engine (scripts/build_engine.sh). +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; + +import { initSync, analyze } from '../public/playground/engine/sqlquack_wasm.js'; +import { rendered, underlined } from '../public/playground/spans.js'; + +initSync({ module: readFileSync(new URL('../public/playground/engine/sqlquack_wasm_bg.wasm', import.meta.url)) }); + +const SCHEMA = JSON.stringify({ ORDERS: [['ORDER_ID', 'NUMBER(38,0)'], ['NOTE', 'VARCHAR']] }); + +const marked = (html) => [...html.matchAll(/]*>(.*?)<\/mark>/g)].map((match) => match[1]); + +test('spans after non-ASCII text underline the characters the diagnostic names', () => { + // 'é' is one UTF-16 unit, '😀' two and 'ß' one: three characters but four units before `nope`. + const sql = "SELECT 'é😀ß' AS note, nope FROM orders"; + const report = JSON.parse(analyze(sql, SCHEMA)); + const [refusal] = report.translate.diagnostics; + assert.equal(refusal.code, 'SQ0004'); + const chars = Array.from(sql); + assert.equal(chars.slice(refusal.span.start, refusal.span.end).join(''), 'nope'); + // Slicing the string by UTF-16 units would land one unit early, on ` nop`. + assert.notEqual(sql.slice(refusal.span.start, refusal.span.end), 'nope'); + const { html } = underlined(chars, [{ kind: 'error', span: refusal.span }]); + assert.deepEqual(marked(html), ['nope']); + const [finding] = report.checks.findings; + assert.equal(chars.slice(finding.span.start, finding.span.end).join(''), 'nope'); +}); + +test('a parse failure on a later line renders at its line and column', () => { + const sql = "SELECT '日本' AS note\nFROM orders\nWHERE"; + const report = JSON.parse(analyze(sql, SCHEMA)); + assert.equal(report.parse.status, 'fail'); + const [failure] = report.parse.diagnostics; + assert.equal(failure.code, 'SQ0001'); + assert.equal(Array.from(sql).slice(failure.span.start, failure.span.end).join(''), 'WHERE'); + assert.match(rendered(failure, Array.from(sql)), /--> query\.sql:3:1\n/); + assert.deepEqual(marked(underlined(Array.from(sql), [{ kind: 'error', span: failure.span }]).html), ['WHERE']); +}); + +test('each stage reports pass, fail or skipped', () => { + const stages = (sql) => { + const report = JSON.parse(analyze(sql, SCHEMA)); + return ['parse', 'checks', 'translate'].map((stage) => report[stage].status); + }; + assert.deepEqual(stages('SELECT order_id FROM orders'), ['pass', 'pass', 'pass']); + assert.deepEqual(stages('SELECT order_id FROM orders WHERE'), ['fail', 'skipped', 'skipped']); + assert.deepEqual(stages('SELECT 1 QUALIFY TRUE'), ['pass', 'fail', 'skipped']); + assert.deepEqual(stages('SELECT HASH(order_id) AS h FROM orders'), ['pass', 'pass', 'fail']); +}); + +test('the policy and session parameters choose the profile, or refuse, as on the command line', () => { + const policy = readFileSync(new URL('../../sqlquack.toml', import.meta.url), 'utf8'); + const translate = (settings) => { + const report = JSON.parse(analyze('SELECT order_id FROM orders', SCHEMA, JSON.stringify(settings))); + return report.settings_error ? { error: report.settings_error } : report.translate; + }; + assert.equal(translate({}).profile, 'the built-in default'); + assert.equal(translate({ policy }).profile, 'utc-v1'); + // Only utc-qiic-v1 records quoted identifiers resolving to upper case. + assert.equal(translate({ policy, session: 'quoted_identifiers_ignore_case = TRUE\n' }).profile, 'utc-qiic-v1'); + // No recorded profile starts the week on Monday: refused with SQ0007, before any conversion. + const monday = translate({ policy, session: 'WEEK_START=1' }); + assert.equal(monday.status, 'fail'); + assert.deepEqual(monday.diagnostics.map((d) => d.code), ['SQ0007']); + assert.equal(monday.statements, undefined); + // The built-in default records no profiles, so any session parameter is refused. + assert.deepEqual(translate({ session: 'TIMEZONE=UTC' }).diagnostics.map((d) => d.code), ['SQ0007']); + assert.match(translate({ policy: 'schema_version = 1\nnot_a_key = 2' }).error, /^policy: /); + assert.match(translate({ session: 'TIMEZONE' }).error, /NAME=VALUE/); +}); + +test('the conversion budget is timed in the browser and refuses with SQ0006 when exceeded', () => { + const policy = readFileSync(new URL('../../sqlquack.toml', import.meta.url), 'utf8'); + // A zero-millisecond budget: any measured conversion time exceeds it. `Instant` would panic here. + const tight = policy.replace(/\[conversion_budget\][^[]*/, '[conversion_budget]\nmilliseconds = 0\nsize_ratio = 2000\nminimum_bytes = 4194304\n\n'); + const report = JSON.parse(analyze('SELECT order_id FROM orders', SCHEMA, JSON.stringify({ policy: tight }))); + assert.equal(report.translate.status, 'fail'); + assert.equal(report.translate.diagnostics[0].code, 'SQ0006'); + assert.match(report.translate.diagnostics[0].label, /beyond the budget of 0 ms/); +}); diff --git a/website/wasm/Cargo.lock b/website/wasm/Cargo.lock index b7d3a53d1..d1890e908 100644 --- a/website/wasm/Cargo.lock +++ b/website/wasm/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" +dependencies = [ + "serde_core", +] + [[package]] name = "shlex" version = "2.0.1" @@ -167,6 +176,14 @@ dependencies = [ "sqlquack-snowflake", ] +[[package]] +name = "sqlquack-policy" +version = "0.2.0" +dependencies = [ + "serde", + "toml", +] + [[package]] name = "sqlquack-rules" version = "0.2.0" @@ -175,6 +192,7 @@ dependencies = [ "serde_json", "sqlquack-core", "sqlquack-duckdb", + "sqlquack-policy", "sqlquack-semantics", "sqlquack-snowflake", "sqlquack-syntax", @@ -228,6 +246,7 @@ dependencies = [ "serde_json", "sqlquack-core", "sqlquack-duckdb", + "sqlquack-policy", "sqlquack-rules", "sqlquack-semantics", "sqlquack-snowflake", @@ -290,6 +309,37 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow 1.0.5", +] + [[package]] name = "unicode-ident" version = "1.0.27" @@ -361,6 +411,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef" + [[package]] name = "zmij" version = "1.0.23" diff --git a/website/wasm/Cargo.toml b/website/wasm/Cargo.toml index 8df66eb53..2ea3af4fd 100644 --- a/website/wasm/Cargo.toml +++ b/website/wasm/Cargo.toml @@ -18,6 +18,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sqlquack-core = { path = "../../crates/sqlquack-core" } sqlquack-duckdb = { path = "../../crates/sqlquack-duckdb", default-features = false } +sqlquack-policy = { path = "../../crates/sqlquack-policy" } sqlquack-rules = { path = "../../crates/sqlquack-rules", default-features = false } sqlquack-semantics = { path = "../../crates/sqlquack-semantics" } sqlquack-snowflake = { path = "../../crates/sqlquack-snowflake" } diff --git a/website/wasm/src/lib.rs b/website/wasm/src/lib.rs index 0c779af84..ddc57a502 100644 --- a/website/wasm/src/lib.rs +++ b/website/wasm/src/lib.rs @@ -4,11 +4,14 @@ //! `sqlquack convert` does, and returns the result as JSON for the page. use std::collections::BTreeMap; +use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sqlquack_core::classification::main::classify::classify; +use sqlquack_core::diagnostics::constants::SESSION_PARAMETERS; use sqlquack_core::diagnostics::main::diagnose::diagnose; +use sqlquack_core::diagnostics::main::unplaced_diagnostic::unplaced_diagnostic; use sqlquack_core::parsing::main::node_location::node_location; use sqlquack_core::parsing::main::parse_source::parse_source; use sqlquack_core::parsing::models::{SourceDialect, SourceLines}; @@ -20,23 +23,29 @@ use sqlquack_core::refusals::models::{Refusal, RefusalCategory}; use sqlquack_duckdb::blocks::main::column_type::column_type; use sqlquack_duckdb::blocks::main::quoted_identifier::quoted_identifier; use sqlquack_duckdb::checking::constants::SEMI_STRUCTURED_HOLDING_TYPE; +use sqlquack_policy::policy::models::Policy; use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; use sqlquack_rules::conversion::main::convert_statement::convert_statement; use sqlquack_rules::conversion::main::quoted_exclusions::quoted_exclusions; +use sqlquack_rules::conversion::main::within_budget::within_budget; use sqlquack_rules::conversion::models::ConversionOptions; +use sqlquack_rules::session::main::parse_session::parse_session; +use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; use sqlquack_semantics::dialects::constants::SNOWFLAKE; use sqlquack_semantics::lineage::main::query_name_findings::query_name_findings; use sqlquack_semantics::lineage::models::{LineageContext, Nullability, SchemaColumn}; use sqlquack_semantics::lineage::types::LineageSchema; -use sqlquack_semantics::types::models::{ColumnSpec, Schema}; -use sqlquack_snowflake::typing::main::cast_target::cast_target; +use sqlquack_semantics::types::main::parse_schema::parse_schema; +use sqlquack_semantics::types::models::Schema; use sqlquack_syntax::{Dialect, Expression, ParseOptions}; use wasm_bindgen::prelude::wasm_bindgen; -/// Every stage's result for `sql` over the relations `schema` declares, as JSON. +/// Every stage's result for `sql` over the relations `schema` declares, as JSON. `settings` is +/// JSON `{"policy": TOML, "session": "NAME=VALUE" lines, "strict": bool, "utc_session": bool}`; +/// an empty policy is the built-in default, as `sqlquack convert --builtin-policy`. #[wasm_bindgen] -pub fn analyze(sql: &str, schema: &str) -> String { - report(sql, schema).to_string() +pub fn analyze(sql: &str, schema: &str, settings: Option) -> String { + report(sql, schema, settings.as_deref().unwrap_or_default()).to_string() } /// `CREATE TABLE` for each relation in `schema`, in the DuckDB types converted SQL reads them as, @@ -72,28 +81,6 @@ pub fn fixture_tables(schema: &str) -> String { json!({ "tables": tables }).to_string() } -/// The relations of a schema file: `{"ORDERS": [["ORDER_ID", "NUMBER(38,0)"], …]}`, or the same -/// under `"relations"`. Columns may also be `{"name": …, "type": …}` objects. -#[derive(Deserialize)] -#[serde(untagged)] -enum RawSchema { - Sections { - relations: BTreeMap>, - }, - Relations(BTreeMap>), -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum RawColumn { - Pair(String, String), - Described { - name: String, - #[serde(rename = "type")] - type_name: String, - }, -} - /// The schema's relations as conversion types them, and as name checks resolve them. #[derive(Debug, Default)] struct Catalog { @@ -107,38 +94,68 @@ impl LineageSchema for Catalog { } } +/// The schema file `text`, read by the same parser as `sqlquack convert --schema`. fn catalog(text: &str) -> Result { if text.trim().is_empty() { return Ok(Catalog::default()); } - let relations = match serde_json::from_str(text).map_err(|error| error.to_string())? { - RawSchema::Sections { relations } | RawSchema::Relations(relations) => relations, + let schema = parse_schema(text).map_err(|error| error.to_string())?; + let columns = schema + .relations + .iter() + .map(|(relation, columns)| { + let named = columns + .iter() + .map(|column| SchemaColumn { + name: column.name.clone(), + quoted: false, + data_type: Some(column.data_type.to_string()), + nullability: Nullability::Unknown, + }) + .collect(); + (relation.clone(), named) + }) + .collect(); + Ok(Catalog { schema, columns }) +} + +/// The playground's policy and session settings. +#[derive(Default, Deserialize)] +#[serde(default)] +struct Settings { + policy: String, + session: String, + strict: bool, + utc_session: bool, +} + +/// Conversion options for `settings` as `sqlquack convert` chooses them, and the matched profile; +/// a session no recorded profile covers is a refusal, as on the command line. +fn chosen_options(settings: &str) -> Result, String> { + let settings: Settings = match settings.trim() { + "" => Settings::default(), + text => serde_json::from_str(text).map_err(|error| error.to_string())?, }; - let log = ProvenanceLog::default(); - let mut catalog = Catalog::default(); - for (relation, raw) in relations { - let mut typed = Vec::with_capacity(raw.len()); - let mut named = Vec::with_capacity(raw.len()); - for column in raw { - let (name, type_name) = match column { - RawColumn::Pair(name, type_name) | RawColumn::Described { name, type_name } => { - (name, type_name) - } - }; - let target = cast_target(&type_name, &log) - .map_err(|error| format!("{relation}.{name}: {error}"))?; - named.push(SchemaColumn { - name: name.clone(), - quoted: false, - data_type: Some(type_name), - nullability: Nullability::Unknown, - }); - typed.push(ColumnSpec::new(name, target.data_type)); - } - catalog.columns.insert(relation.clone(), named); - catalog.schema.relations.insert(relation, typed); - } - Ok(catalog) + let policy = match settings.policy.trim() { + "" => None, + text => Some( + text.parse::() + .map_err(|error| format!("policy: {error}"))?, + ), + }; + let lines: Vec = settings + .session + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect(); + let parameters = parse_session(&lines)?; + Ok(session_conversion_options( + policy.as_ref(), + (settings.strict, settings.utc_session), + ¶meters, + )) } /// A stage's verdict and what it found. @@ -186,11 +203,15 @@ fn placed(mut refusal: Refusal, statement: &Expression, lines: &SourceLines) -> refusal } -fn report(sql: &str, schema: &str) -> Value { +fn report(sql: &str, schema: &str, settings: &str) -> Value { let catalog = match catalog(schema) { Ok(catalog) => catalog, Err(error) => return json!({ "schema_error": error }), }; + let session = match chosen_options(settings) { + Ok(session) => session, + Err(error) => return json!({ "settings_error": error }), + }; let lines = SourceLines::new(sql); // 1. Parse as Snowflake. @@ -249,11 +270,20 @@ fn report(sql: &str, schema: &str) -> Value { return json!({ "parse": parse, "checks": checks, "translate": Stage::skipped() }); } - // 3. Convert each statement, or refuse it, as `sqlquack convert --builtin-policy` does. + // 3. Convert each statement, or refuse it, as `sqlquack convert` does under the same policy. let policy = builtin_catalogue(); + let (options, profile) = match session { + Ok(chosen) => chosen, + Err(reason) => { + let refused = unplaced_diagnostic(SESSION_PARAMETERS, reason) + .and_then(|diagnostic| serde_json::to_value(diagnostic).ok()); + let translate = Stage::new(false, refused.into_iter().collect()); + return json!({ "parse": parse, "checks": checks, "translate": translate }); + } + }; let options = ConversionOptions { quoted_exclusions: quoted_exclusions(sql), - ..ConversionOptions::default() + ..options }; let mut refusals = Vec::new(); let mut converted = Vec::new(); @@ -267,11 +297,14 @@ fn report(sql: &str, schema: &str) -> Value { refusals.extend(declared); continue; } - match convert_statement( + let started = now_ms(); + let done = convert_statement( statement, (&catalog.schema, &options, &lines), ProvenanceLog::default(), - ) { + ); + let elapsed = Duration::from_secs_f64((now_ms() - started).max(0.0) / 1000.0); + match done.and_then(|done| within_budget(done, (sql, elapsed), &options)) { Ok(done) => converted.push(json!({ "sql": done.sql, "columns": done.columns.iter().map(|column| json!({ @@ -284,5 +317,26 @@ fn report(sql: &str, schema: &str) -> Value { } let mut translate = Stage::new(refusals.is_empty(), diagnostics(&refusals, sql)); translate.extra.insert("statements", json!(converted)); + translate.extra.insert("profile", json!(profile)); json!({ "parse": parse, "checks": checks, "translate": translate }) } + +/// Milliseconds on a monotonic clock: `performance.now()` in the browser, where `Instant` panics. +#[cfg(target_arch = "wasm32")] +fn now_ms() -> f64 { + #[wasm_bindgen] + extern "C" { + #[wasm_bindgen(js_namespace = performance, js_name = now)] + fn performance_now() -> f64; + } + performance_now() +} + +/// Milliseconds on a monotonic clock, for native builds of this crate. +#[cfg(not(target_arch = "wasm32"))] +fn now_ms() -> f64 { + use std::sync::OnceLock; + use std::time::Instant; + static START: OnceLock = OnceLock::new(); + START.get_or_init(Instant::now).elapsed().as_secs_f64() * 1000.0 +}