Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions crates/sqlquack-core/src/diagnostics/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,9 @@ pub const CODES: &[CodeEntry] = &[
alias: None,
title: "the conversion needs a known divergence the policy refuses",
help: Some(
"allow the divergence under [divergences.<id>] in the policy file (--policy), drop \
--strict or --loud-divergences-only, or confirm its assumption (--utc-session, \
--confirm-declared-offsets, --python-runtime); --builtin-policy refuses every one",
"the help names the divergence and the fixes that work in the current mode: allow \
it in the policy file, drop --strict or --loud-divergences-only, or confirm its \
assumption; --builtin-policy refuses every one",
),
explanation: "The construct at the span converts only by relying on the known \
divergence the label names, where DuckDB may raise or differ from Snowflake as the \
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlquack-core/src/refusals/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
//! The wording refusals share with the code that reads them back.

/// The words before the divergence id in a refused divergence's reason.
pub const NEEDS_DIVERGENCE: &str = " needs known divergence `";
3 changes: 2 additions & 1 deletion crates/sqlquack-core/src/refusals/main/refused_divergence.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
//! `refused_divergence`: the refusal of a construct whose conversion needs a refused divergence.

use crate::classification::models::Construct;
use crate::refusals::constants::NEEDS_DIVERGENCE;
use crate::refusals::models::{Refusal, RefusalCategory};

/// The construct converts only by relying on `divergence`, which the policy refuses in this mode.
pub fn refused_divergence(construct: Construct, divergence: &str) -> Refusal {
Refusal {
reason: format!(
"{} needs known divergence `{divergence}`, which the policy refuses in this mode",
"{}{NEEDS_DIVERGENCE}{divergence}`, which the policy refuses in this mode",
construct.key()
),
construct: Some(construct),
Expand Down
1 change: 1 addition & 0 deletions crates/sqlquack-core/src/refusals/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Structured refusals: what SQLQuack will not convert, where, and why.

pub(crate) mod _helpers;
pub mod constants;
pub mod main;
pub mod models;
pub mod types;
Expand Down
12 changes: 12 additions & 0 deletions crates/sqlquack-core/src/refusals/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use serde::Serialize;

use crate::classification::models::Construct;
use crate::parsing::models::SourceLocation;
use crate::refusals::constants::NEEDS_DIVERGENCE;

/// Why a construct or statement is refused.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
Expand Down Expand Up @@ -64,6 +65,17 @@ pub struct Refusal {
pub reason: String,
}

impl Refusal {
/// The id of the known divergence a refused divergence needs; none for other refusals.
pub fn divergence(&self) -> Option<&str> {
if self.category != RefusalCategory::DivergenceRefused {
return None;
}
let (_, rest) = self.reason.split_once(NEEDS_DIVERGENCE)?;
rest.split_once('`').map(|(id, _)| id)
}
}

/// What the rule catalogue says about one construct.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Disposition {
Expand Down
64 changes: 62 additions & 2 deletions crates/sqlquack-policy/src/policy/_helpers/parsing.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,76 @@
//! Policy text to a validated policy.

use crate::policy::_helpers::validation::problems;
use crate::policy::constants::{DIVERGENCE_FIELDS, MAX_SPELLING_DISTANCE};
use crate::policy::errors::PolicyError;
use crate::policy::models::Policy;

pub(crate) fn parse_policy(text: &str) -> Result<Policy, PolicyError> {
let policy: Policy =
toml::from_str(text).map_err(|error| PolicyError::Schema(error.to_string()))?;
let policy: Policy = toml::from_str(text).map_err(|error| {
PolicyError::Schema(match text.parse::<toml::Table>() {
Ok(table) => incomplete_divergence(&table).unwrap_or_else(|| error.to_string()),
Err(_not_toml) => error.to_string(),
})
})?;
let found = problems(&policy);
if found.is_empty() {
Ok(policy)
} else {
Err(PolicyError::Invalid(found))
}
}

/// Why a `[divergences.<id>]` table lacking fields fails: a misspelt id, or a partial entry.
fn incomplete_divergence(table: &toml::Table) -> Option<String> {
let divergences = table.get("divergences")?.as_table()?;
let missing = |entry: &toml::Value| -> Vec<&str> {
DIVERGENCE_FIELDS
.iter()
.copied()
.filter(|field| entry.get(field).is_none())
.collect()
};
let (id, absent) = divergences
.iter()
.map(|(id, entry)| (id, missing(entry)))
.find(|(_, absent)| !absent.is_empty())?;
let nearest = divergences
.iter()
.filter(|(other, entry)| *other != id && missing(entry).is_empty())
.map(|(other, _)| (edit_distance(id, other), other))
.filter(|(distance, _)| *distance <= MAX_SPELLING_DISTANCE)
.min();
Some(match nearest {
Some((_, known)) => {
format!("unknown divergence `{id}` in [divergences.{id}]; did you mean `{known}`?")
}
None => format!(
"[divergences.{id}] is missing field{} {}: a policy file declares each divergence in \
full, so copy its entry from the repository's sqlquack.toml, or check the id's \
spelling",
if absent.len() == 1 { "" } else { "s" },
absent
.iter()
.map(|field| format!("`{field}`"))
.collect::<Vec<_>>()
.join(", ")
),
})
}

/// The Levenshtein distance between `left` and `right`, by character.
fn edit_distance(left: &str, right: &str) -> usize {
let right: Vec<char> = right.chars().collect();
let mut previous: Vec<usize> = (0..=right.len()).collect();
for (row, left_char) in left.chars().enumerate() {
let mut current = vec![row + 1; right.len() + 1];
for (column, right_char) in right.iter().enumerate() {
let substitution = previous[column] + usize::from(left_char != *right_char);
current[column + 1] = substitution
.min(previous[column + 1] + 1)
.min(current[column] + 1);
}
previous = current;
}
previous[right.len()]
}
6 changes: 6 additions & 0 deletions crates/sqlquack-policy/src/policy/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ pub const DEFAULT_SESSION_PROFILE: &str = "utc-v1";
/// The only schema version this build understands.
pub const SCHEMA_VERSION: u32 = 1;

/// The fields every `[divergences.<id>]` entry must give.
pub const DIVERGENCE_FIELDS: [&str; 5] = ["constructs", "default", "strict", "reason", "failure"];

/// The most character edits between a misspelt divergence id and the one it suggests.
pub const MAX_SPELLING_DISTANCE: usize = 2;

/// HASH values differ from Snowflake's; equal inputs hash equally within one run.
pub const HASH_DIVERGENCE: &str = "hash";
/// The divergence where DuckDB raises for a nanosecond timestamp outside its range.
Expand Down
16 changes: 16 additions & 0 deletions crates/sqlquack-policy/src/policy/tests/test_parse_policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,22 @@ fn given_policy_text_when_parsing_then_schema_and_rules_are_enforced() {
with: "schema_version = 1\noverrides = true",
expected_error: Some("unknown field `overrides`"),
},
PolicyTestCase {
description: "a misspelt divergence id names the one it is near",
replace: "failure = \"value\"\n",
with: "failure = \"value\"\n\n[divergences.hsah]\ndefault = \"warn\"\n",
expected_error: Some(
"unknown divergence `hsah` in [divergences.hsah]; did you mean `hash`?",
),
},
PolicyTestCase {
description: "a partial divergence entry names the fields it lacks",
replace: "failure = \"value\"\n",
with: "failure = \"value\"\n\n[divergences.json_fidelity]\ndefault = \"warn\"\n",
expected_error: Some(
"[divergences.json_fidelity] is missing fields `constructs`, `strict`, `reason`, `failure`",
),
},
PolicyTestCase {
description: "an unknown comparator key is rejected",
replace: "timestamp_tz = \"instant\"",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ use std::collections::BTreeMap;
use crate::conversion::models::ConversionOptions;
use sqlquack_policy::policy::constants::SESSION_TIMEZONE_UTC;
use sqlquack_policy::policy::constants::{
DEFAULT_SESSION_PROFILE, QUOTED_IDENTIFIERS_IGNORE_CASE, SESSION_TIME_ZONE, TIMEZONE_PARAMETER,
DEFAULT_SESSION_PROFILE, QUOTED_IDENTIFIERS_IGNORE_CASE, RESULT_NEUTRAL_SESSION_PARAMETERS,
SESSION_TIME_ZONE, TIMEZONE_PARAMETER,
};
use sqlquack_policy::policy::models::{Policy, SessionValue};

Expand All @@ -21,13 +22,19 @@ pub fn session_conversion_options(
parameters: &BTreeMap<String, String>,
) -> Result<(ConversionOptions, String), String> {
let Some(policy) = policy else {
return match parameters.is_empty() {
let unknown: Vec<&str> = parameters
.keys()
.map(String::as_str)
.filter(|name| !builtin_accepts(name))
.collect();
return match unknown.is_empty() {
true => Ok((ConversionOptions::default(), BUILTIN_PROFILE.to_owned())),
false => Err(
"the built-in default records no session profiles, so it takes no session \
parameters"
.to_owned(),
),
false => Err(format!(
"the built-in default records no session profiles, so it takes only TIMEZONE and \
the result-neutral parameters, not {}; pass --policy with a policy file whose \
session profiles cover them, such as the repository's sqlquack.toml",
unknown.join(", ")
)),
};
};
let profile = matching_profile(
Expand All @@ -53,3 +60,11 @@ pub fn session_conversion_options(
}
Ok((options, profile))
}

/// Whether the built-in default, which records no session profiles, takes parameter `name`.
fn builtin_accepts(name: &str) -> bool {
name == TIMEZONE_PARAMETER
|| RESULT_NEUTRAL_SESSION_PARAMETERS
.iter()
.any(|(neutral, _)| *neutral == name)
}
1 change: 1 addition & 0 deletions crates/sqlquack-semantics/src/types/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pub(crate) mod schema_file;
pub(crate) mod schema_shape;
82 changes: 82 additions & 0 deletions crates/sqlquack-semantics/src/types/_helpers/schema_shape.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
//! What is wrong with a schema file's shape, in its own terms, where JSON decoding fails.

use serde_json::{Map, Value};

use crate::types::constants::{COLUMN_KEYS, REQUIRED_COLUMN_KEYS, SCHEMA_SECTIONS};

/// The first shape problem of the schema `top`, naming the key, relation or column at fault.
pub(crate) fn shape_problem(top: &Value) -> Option<String> {
let Value::Object(top) = top else {
return Some(
"the schema must be a JSON object: relations by name, or `relations` and `functions`"
.to_owned(),
);
};
let sectioned = SCHEMA_SECTIONS
.iter()
.any(|section| top.contains_key(*section));
if sectioned
&& let Some(key) = top
.keys()
.find(|key| !SCHEMA_SECTIONS.contains(&key.as_str()))
{
return Some(format!(
"unknown top-level key `{key}`; a schema with sections holds only `relations` and \
`functions`"
));
}
let relations = match sectioned {
true => top.get("relations")?.as_object()?,
false => top,
};
relations
.iter()
.find_map(|(relation, columns)| relation_problem(relation, columns))
}

/// What is wrong with `relation`'s columns, if anything.
fn relation_problem(relation: &str, columns: &Value) -> Option<String> {
let Value::Array(columns) = columns else {
return Some(format!(
"relation {relation} must list its columns, as [[\"NAME\", \"TYPE\"], …]"
));
};
columns.iter().enumerate().find_map(|(index, column)| {
column_problem(
&format!("relation {relation}, column {}", index + 1),
column,
)
})
}

/// What is wrong with the column at `place`, if anything.
fn column_problem(place: &str, column: &Value) -> Option<String> {
match column {
Value::Array(pair) => match pair.as_slice() {
[Value::String(_), Value::String(_)] => None,
[Value::String(name)] => Some(format!("{place} ({name}) has no type")),
_ => Some(format!("{place} must be [\"NAME\", \"TYPE\"], two strings")),
},
Value::Object(fields) => object_column_problem(place, fields),
_ => Some(format!(
"{place} must be [\"NAME\", \"TYPE\"] or {{\"name\": …, \"type\": …}}"
)),
}
}

/// What is wrong with the column at `place` written as an object, if anything.
fn object_column_problem(place: &str, fields: &Map<String, Value>) -> Option<String> {
if let Some(key) = fields
.keys()
.find(|key| !COLUMN_KEYS.contains(&key.as_str()))
{
return Some(format!(
"{place} has unknown key `{key}`; a column takes {}",
COLUMN_KEYS.join(", ")
));
}
REQUIRED_COLUMN_KEYS
.iter()
.find(|key| !fields.get(**key).is_some_and(Value::is_string))
.map(|key| format!("{place} needs `{key}` as a string"))
}
9 changes: 9 additions & 0 deletions crates/sqlquack-semantics/src/types/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,12 @@

/// The offsets a schema may declare for a TIMESTAMP_TZ source column, all meaning +00:00.
pub const DECLARABLE_OFFSETS: &[&str] = &["+00:00", "UTC"];

/// The top-level keys of a schema file written in sections.
pub const SCHEMA_SECTIONS: [&str; 2] = ["relations", "functions"];

/// The keys a column written as an object may hold.
pub const COLUMN_KEYS: [&str; 4] = ["name", "type", "typed_leaves", "offset"];

/// The keys a column written as an object must give, as strings.
pub const REQUIRED_COLUMN_KEYS: [&str; 2] = ["name", "type"];
9 changes: 7 additions & 2 deletions crates/sqlquack-semantics/src/types/main/parse_schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,18 @@ use std::collections::BTreeMap;
use sqlquack_core::provenance::models::ProvenanceLog;

use crate::types::_helpers::schema_file::{RawSchema, typed_columns, typed_function};
use crate::types::_helpers::schema_shape::shape_problem;
use crate::types::errors::SchemaError;
use crate::types::models::Schema;

/// The schema `text` declares; a relation or function using a type SQLQuack does not know is left out.
pub fn parse_schema(text: &str) -> Result<Schema, SchemaError> {
let raw: RawSchema =
serde_json::from_str(text).map_err(|error| SchemaError::Shape(error.to_string()))?;
let raw: RawSchema = serde_json::from_str(text).map_err(|error| {
SchemaError::Shape(match serde_json::from_str::<serde_json::Value>(text) {
Ok(top) => shape_problem(&top).unwrap_or_else(|| error.to_string()),
Err(_not_json) => error.to_string(),
})
})?;
let (relations, functions) = match raw {
RawSchema::Sections(sections) => (sections.relations, sections.functions),
RawSchema::Relations(relations) => (relations, BTreeMap::new()),
Expand Down
11 changes: 8 additions & 3 deletions crates/sqlquack/src/cli/_helpers/convert_command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@ use crate::analysis::models::Analysis;
use crate::cli::_helpers::evidence::caller_choices;
use crate::cli::models::{
CheckArguments, CheckReport, ConvertArguments, ConvertRefusal, ConvertReport, ExplainArguments,
OverrideListing,
OverrideListing, PolicyArguments,
};

use crate::cli::_helpers::evidence::with_divergence_help;
use crate::cli::main::convert_report::convert_report;

/// The path naming standard input.
Expand Down Expand Up @@ -257,7 +258,7 @@ pub(crate) fn analysis_report(
analysis: &Analysis,
sql: &str,
(schema, options, policy): (&Schema, &ConversionOptions, Option<&Policy>),
(parameters, profile): (&BTreeMap<String, String>, String),
(arguments, parameters, profile): (&PolicyArguments, &BTreeMap<String, String>, String),
) -> Result<ConvertReport, String> {
let converted = analysis.parsed && analysis.refusals.is_empty();
let provenance = merge_provenance(analysis.conversions.iter().map(|done| &done.provenance));
Expand Down Expand Up @@ -299,7 +300,11 @@ pub(crate) fn analysis_report(
refusals: analysis
.refusals
.iter()
.zip(diagnose(&analysis.refusals, sql))
.zip(with_divergence_help(
&analysis.refusals,
diagnose(&analysis.refusals, sql),
(policy, arguments, parameters),
))
.map(convert_refusal)
.collect(),
python_functions: python_functions(&provenance.divergences, schema)?,
Expand Down
Loading
Loading