Skip to content

Commit e8583f3

Browse files
kvlongekvlonge
committed
Merge remote-tracking branch 'origin/main' into feat/output-names-receipts
2 parents 264c798 + 596399e commit e8583f3

82 files changed

Lines changed: 2056 additions & 113 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ A `VALUES` list of more than 1,000 rows is refused for now.
107107
session parameters yet and convert under `utc-v1`.
108108

109109
`convert`, `function`, `census` and `index` read `sqlquack.toml` (or `--policy`); without one they
110-
exit 2 unless `--builtin-policy` asks for the built-in default, which refuses every known divergence.
110+
use the built-in policy, which refuses every known divergence, as `--builtin-policy` asks for.
111111
They share the policy flags (`--policy`, `--builtin-policy`, `--strict`, `--utc-session`,
112112
`--confirm-declared-offsets`, `--python-runtime` and `--loud-divergences-only`, which relies only
113113
on divergences whose converted SQL raises rather than returns another value). Every command with a report takes

‎cases/translation/functions/flatten.path.toml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
category = "translation"
2+
construct = "sf.functions.flatten"
3+
session_profile = "utc-v1"
4+
5+
[[case]]
6+
id = "t001_object_under_path"
7+
description = "PATH reaching an object: one row per key, with KEY, VALUE and a PATH prefixed with the given path, for a plain key and one with a space. Catches: the prefix joined without a dot, or a quoted key written as a plain one"
8+
columns = [{ name = "a", type = "VARCHAR" }]
9+
rows = [
10+
["'{\"x\":{\"y\":{\"b\":1,\"a c\":2}}}'"],
11+
]
12+
query = "SELECT i.ordinal, f.key, f.index, f.value, f.path FROM input i, LATERAL FLATTEN(INPUT => PARSE_JSON(i.a), PATH => 'x.y') f ORDER BY i.ordinal, f.key"
13+
order = "ordered"
14+
tags = ["type-combination"]
15+
16+
[[case]]
17+
id = "t002_path_key_case"
18+
description = "PATH keys match case-sensitively: 'Items' finds nothing where the key is items, and one-step 'items' reaches the array. Catches: keys matched ignoring case"
19+
columns = [{ name = "a", type = "VARCHAR" }]
20+
rows = [
21+
["'{\"items\":[10,20]}'"],
22+
]
23+
query = "SELECT 'lower' AS k, f.index, f.value, f.path FROM input i, LATERAL FLATTEN(INPUT => PARSE_JSON(i.a), PATH => 'items') f UNION ALL SELECT 'upper', f.index, f.value, f.path FROM input i, LATERAL FLATTEN(INPUT => PARSE_JSON(i.a), PATH => 'Items') f ORDER BY 1, 2"
24+
order = "ordered"
25+
tags = ["type-combination"]
26+
27+
[[case]]
28+
id = "t003_scalar_and_null_at_path"
29+
description = "PATH reaching a scalar, a JSON null and an empty array gives no rows. Catches: a scalar flattened into one row, or a JSON null giving a row"
30+
columns = [{ name = "a", type = "VARCHAR" }]
31+
rows = [
32+
["'{\"x\":5}'"],
33+
["'{\"x\":null}'"],
34+
["'{\"x\":[]}'"],
35+
["'{\"x\":[7]}'"],
36+
]
37+
query = "SELECT i.ordinal, f.index, f.value, f.path FROM input i, LATERAL FLATTEN(INPUT => PARSE_JSON(i.a), PATH => 'x') f ORDER BY i.ordinal, f.index"
38+
order = "ordered"
39+
tags = ["type-combination"]

‎cases/translation/functions/pow.power-of-two.toml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
category = "translation"
2+
construct = "sf.functions.pow.power"
3+
session_profile = "utc-v1"
4+
5+
[[case]]
6+
id = "t001_whole_exponents"
7+
description = "POWER(2, n) for whole exponents from -52 to 64, as dbt_utils.date_spine writes it for each bit. Catches: an inexact power of two, an integer result, or a lost low bit past 2^53"
8+
columns = [{ name = "n", type = "NUMBER(38,0)" }]
9+
rows = [
10+
["0"], ["1"], ["2"], ["3"], ["4"], ["5"], ["6"], ["7"], ["8"], ["9"], ["10"], ["11"],
11+
["31"], ["32"], ["52"], ["53"], ["62"], ["63"], ["64"],
12+
["-1"], ["-2"], ["-52"],
13+
]
14+
query = "SELECT POWER(2, n) AS r FROM input ORDER BY ordinal"
15+
order = "ordered"
16+
tags = ["type-combination"]
17+
18+
[[case]]
19+
id = "t002_literal_exponents_in_a_sum"
20+
description = "The dbt_utils.date_spine shape: a NUMBER times POWER(2, k) for literal k, summed, plus 1. Catches: the result typed NUMBER instead of FLOAT, or a power other than 2^k"
21+
columns = [{ name = "a", type = "NUMBER(38,0)" }, { name = "b", type = "NUMBER(38,0)" }]
22+
rows = [
23+
["0", "0"],
24+
["1", "0"],
25+
["1", "1"],
26+
["NULL", "1"],
27+
]
28+
query = "SELECT a * POWER(2, 0) + b * POWER(2, 1) + 1 AS r FROM input ORDER BY ordinal"
29+
order = "ordered"
30+
tags = ["type-combination"]
31+
32+
[[case]]
33+
id = "t003_range_edges"
34+
description = "The largest and smallest finite powers of two in a double, 2^1023 and 2^-1074, and 2^-1075 below the smallest. Catches: 2^1023 overflowing, or the subnormal 2^-1074 flushed to 0"
35+
columns = [{ name = "n", type = "NUMBER(38,0)" }]
36+
rows = [["1023"], ["-1074"], ["-1075"]]
37+
query = "SELECT POWER(2, n) AS r FROM input ORDER BY ordinal"
38+
order = "ordered"
39+
tags = ["type-combination"]
40+
41+
[[case]]
42+
id = "t004_overflow"
43+
description = "2^1024, past the largest double. Catches: a finite value or an error where Snowflake gives its overflow result"
44+
columns = [{ name = "n", type = "NUMBER(38,0)" }]
45+
rows = [["1024"]]
46+
query = "SELECT POWER(2, n) AS r FROM input ORDER BY ordinal"
47+
order = "ordered"
48+
tags = ["type-combination"]
49+
50+
[[case]]
51+
id = "t005_null_exponent"
52+
description = "POWER(2, NULL). Catches: 1 or 2 returned instead of NULL"
53+
columns = [{ name = "n", type = "NUMBER(38,0)" }]
54+
rows = [["NULL"], ["3"]]
55+
query = "SELECT POWER(2, n) AS r FROM input ORDER BY ordinal"
56+
order = "ordered"
57+
tags = ["null"]

‎cases/translation/functions/regexp-substr.empty-match-position.toml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
category = "translation"
2+
construct = "sf.functions.regexp-substr"
3+
session_profile = "utc-v1"
4+
5+
[[case]]
6+
id = "t001_empty_match_at_and_past_the_end"
7+
description = "REGEXP_SUBSTR(a, 'x*', n) on 'abc' with n at the last character, one past it and two past it, where 'x*' matches the empty string. Catches: '' where Snowflake gives NULL, or an error past the end where Snowflake answers"
8+
columns = [{ name = "a", type = "VARCHAR" }, { name = "n", type = "NUMBER(38,0)" }]
9+
rows = [
10+
["'abc'", "3"],
11+
["'abc'", "4"],
12+
["'abc'", "5"],
13+
]
14+
query = "SELECT n, REGEXP_SUBSTR(a, 'x*', n) AS r FROM input ORDER BY ordinal"
15+
order = "ordered"
16+
tags = ["type-combination"]

‎cases/translation/functions/to-char.month-names.toml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
category = "translation"
2+
construct = "sf.functions.to-char.to-char"
3+
session_profile = "utc-v1"
4+
5+
[[case]]
6+
id = "t001_mon_every_month"
7+
description = "TO_CHAR(date, 'MON') for a date in every month, as dbt_date.get_date_dimension writes month_name_short. Catches: the month number written, the wrong letter case, or a locale's abbreviation"
8+
columns = [{ name = "d", type = "DATE" }]
9+
rows = [
10+
["'2024-01-15'"], ["'2024-02-29'"], ["'2024-03-15'"], ["'2024-04-15'"], ["'2024-05-15'"], ["'2024-06-15'"],
11+
["'2024-07-15'"], ["'2024-08-15'"], ["'2024-09-15'"], ["'2024-10-15'"], ["'2024-11-15'"], ["'2024-12-31'"],
12+
]
13+
query = "SELECT TO_CHAR(d, 'MON') AS r FROM input ORDER BY ordinal"
14+
order = "ordered"
15+
tags = ["type-combination"]
16+
17+
[[case]]
18+
id = "t002_mmmm_every_month"
19+
description = "TO_CHAR(date, 'MMMM') for a date in every month, as dbt_date.get_date_dimension writes month_name. Catches: MMMM read as MM twice ('0101'), the wrong letter case, or an abbreviation"
20+
columns = [{ name = "d", type = "DATE" }]
21+
rows = [
22+
["'2024-01-15'"], ["'2024-02-29'"], ["'2024-03-15'"], ["'2024-04-15'"], ["'2024-05-15'"], ["'2024-06-15'"],
23+
["'2024-07-15'"], ["'2024-08-15'"], ["'2024-09-15'"], ["'2024-10-15'"], ["'2024-11-15'"], ["'2024-12-31'"],
24+
]
25+
query = "SELECT TO_CHAR(d, 'MMMM') AS r FROM input ORDER BY ordinal"
26+
order = "ordered"
27+
tags = ["type-combination"]
28+
29+
[[case]]
30+
id = "t003_month_names_in_a_format"
31+
description = "MON and MMMM beside other fields and on a timestamp. Catches: a field next to the name misread, or the timestamp's time leaking into the name"
32+
columns = [{ name = "d", type = "DATE" }, { name = "t", type = "TIMESTAMP_NTZ(9)" }]
33+
rows = [
34+
["'2024-09-05'", "'2024-05-01 23:59:59'"],
35+
]
36+
query = "SELECT TO_CHAR(d, 'DD MON YYYY') AS a, TO_CHAR(d, 'MMMM DD, YYYY') AS b, TO_CHAR(t, 'MON') AS c, TO_CHAR(t, 'MMMM') AS e FROM input ORDER BY ordinal"
37+
order = "ordered"
38+
tags = ["type-combination"]
39+
40+
[[case]]
41+
id = "t004_null"
42+
description = "A NULL date. Catches: a month name for NULL"
43+
columns = [{ name = "d", type = "DATE" }]
44+
rows = [["NULL"], ["'2024-03-01'"]]
45+
query = "SELECT TO_CHAR(d, 'MON') AS a, TO_CHAR(d, 'MMMM') AS b FROM input ORDER BY ordinal"
46+
order = "ordered"
47+
tags = ["null"]
48+
49+
[[case]]
50+
id = "t007_names_beside_month_and_day"
51+
description = "MONMM and MMMMDD, a name directly before another field. Catches: the name's letters read as part of the next field"
52+
columns = [{ name = "d", type = "DATE" }]
53+
rows = [["'2024-09-05'"]]
54+
query = "SELECT TO_CHAR(d, 'MONMM') AS a, TO_CHAR(d, 'MMMMDD') AS b FROM input ORDER BY ordinal"
55+
order = "ordered"
56+
tags = ["type-combination"]

‎coverage/translation/results-baseline.json‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6926,7 +6926,7 @@
69266926
"semi-structured/flatten_outer.toml#flatten_top_level_outer_plain": "matching",
69276927
"semi-structured/flatten_outer.toml#flatten_top_level_outer_recursive": "refused",
69286928
"semi-structured/flatten_outer.toml#lateral_subquery_without_rows": "matching",
6929-
"semi-structured/flatten_recursive.toml#flatten_path_option": "refused",
6929+
"semi-structured/flatten_recursive.toml#flatten_path_option": "matching",
69306930
"semi-structured/flatten_recursive.toml#flatten_recursive": "matching",
69316931
"semi-structured/flatten_recursive.toml#flatten_recursive_deep": "matching",
69326932
"semi-structured/flatten_recursive.toml#flatten_recursive_filtered": "matching",
@@ -9395,8 +9395,11 @@
93959395
"translation/functions/first-value.toml#t002_ignore_nulls": "matching",
93969396
"translation/functions/first-value.toml#t003_frame": "matching",
93979397
"translation/functions/first-value.toml#t004_null_expr": "matching",
9398+
"translation/functions/flatten.path.toml#t001_object_under_path": "matching",
9399+
"translation/functions/flatten.path.toml#t002_path_key_case": "matching",
9400+
"translation/functions/flatten.path.toml#t003_scalar_and_null_at_path": "matching",
93989401
"translation/functions/flatten.toml#t001_default": "matching",
9399-
"translation/functions/flatten.toml#t002_path": "refused",
9402+
"translation/functions/flatten.toml#t002_path": "matching",
94009403
"translation/functions/flatten.toml#t003_outer": "matching",
94019404
"translation/functions/flatten.toml#t004_recursive": "matching",
94029405
"translation/functions/flatten.toml#t005_mode": "matching",
@@ -10235,6 +10238,11 @@
1023510238
"translation/functions/pow.pow.toml#t007_overflow_inf": "refused",
1023610239
"translation/functions/pow.pow.toml#t008_neg_base": "refused",
1023710240
"translation/functions/pow.pow.toml#t009_neg_frac": "refused",
10241+
"translation/functions/pow.power-of-two.toml#t001_whole_exponents": "matching",
10242+
"translation/functions/pow.power-of-two.toml#t002_literal_exponents_in_a_sum": "matching",
10243+
"translation/functions/pow.power-of-two.toml#t003_range_edges": "matching",
10244+
"translation/functions/pow.power-of-two.toml#t004_overflow": "matching",
10245+
"translation/functions/pow.power-of-two.toml#t005_null_exponent": "matching",
1023810246
"translation/functions/pow.power.toml#t001_int": "refused",
1023910247
"translation/functions/pow.power.toml#t002_dec": "refused",
1024010248
"translation/functions/pow.power.toml#t003_float": "refused",
@@ -10327,6 +10335,7 @@
1032710335
"translation/functions/regexp-substr-all.toml#t009_null_regex_parameters": "refused",
1032810336
"translation/functions/regexp-substr-all.toml#t010_null_group_num": "refused",
1032910337
"translation/functions/regexp-substr-all.toml#t011_empty": "matching",
10338+
"translation/functions/regexp-substr.empty-match-position.toml#t001_empty_match_at_and_past_the_end": "refused",
1033010339
"translation/functions/regexp-substr.toml#t001_varchar": "matching",
1033110340
"translation/functions/regexp-substr.toml#t002_position_occurrence": "refused",
1033210341
"translation/functions/regexp-substr.toml#t003_group": "refused",
@@ -11162,13 +11171,18 @@
1116211171
"translation/functions/to-boolean.toml#t003_invalid_string": "matching_error",
1116311172
"translation/functions/to-boolean.toml#t004_nan": "refused",
1116411173
"translation/functions/to-boolean.toml#t005_null_string_or_numeric_expr": "matching",
11174+
"translation/functions/to-char.month-names.toml#t001_mon_every_month": "matching",
11175+
"translation/functions/to-char.month-names.toml#t002_mmmm_every_month": "matching",
11176+
"translation/functions/to-char.month-names.toml#t003_month_names_in_a_format": "refused",
11177+
"translation/functions/to-char.month-names.toml#t004_null": "matching",
11178+
"translation/functions/to-char.month-names.toml#t007_names_beside_month_and_day": "refused",
1116511179
"translation/functions/to-char.to-char.toml#t001_number_format": "refused",
11166-
"translation/functions/to-char.to-char.toml#t002_date_format": "refused",
11180+
"translation/functions/to-char.to-char.toml#t002_date_format": "matching",
1116711181
"translation/functions/to-char.to-char.toml#t003_binary_format": "refused",
1116811182
"translation/functions/to-char.to-char.toml#t004_variant": "refused",
1116911183
"translation/functions/to-char.to-char.toml#t005_null_expr": "matching",
1117011184
"translation/functions/to-char.to-varchar.toml#t001_number_format": "refused",
11171-
"translation/functions/to-char.to-varchar.toml#t002_date_format": "refused",
11185+
"translation/functions/to-char.to-varchar.toml#t002_date_format": "matching",
1117211186
"translation/functions/to-char.to-varchar.toml#t003_binary_format": "refused",
1117311187
"translation/functions/to-char.to-varchar.toml#t004_variant": "matching",
1117411188
"translation/functions/to-char.to-varchar.toml#t005_null_expr": "matching",

‎crates/sqlquack-core/src/diagnostics/constants.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ pub const CODES: &[CodeEntry] = &[
155155
help: Some(
156156
"the help names the divergence and the fixes that work in the current mode: allow \
157157
it in the policy file, drop --strict or --loud-divergences-only, or confirm its \
158-
assumption; --builtin-policy refuses every one",
158+
assumption; the built-in policy refuses every one",
159159
),
160160
explanation: "The construct at the span converts only by relying on the known \
161161
divergence the label names, where DuckDB may raise or differ from Snowflake as the \

‎crates/sqlquack-evidence/src/cases/_helpers/loading.rs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use sqlquack_policy::policy::models::{Policy, SessionValue};
77

88
use crate::cases::_helpers::functions::{create_function, function_problems};
99
use crate::cases::_helpers::hashing::case_hash;
10+
use crate::cases::_helpers::setup::{profile_session_problems, setup_problems};
1011
use crate::cases::_helpers::source_sql::{row_sources, source_sql};
1112
use crate::cases::models::{
1213
CaseDefinition, CaseFile, CaseInput, CaseRole, EvidenceKind, HashedContent, LoadedCase,
@@ -49,9 +50,23 @@ pub(crate) fn load_file(
4950
.iter()
5051
.flat_map(|case| function_problems(&format!("{relative}#{}", case.id), case)),
5152
);
53+
problems.extend(
54+
file.cases
55+
.iter()
56+
.flat_map(|case| setup_problems(&format!("{relative}#{}", case.id), case)),
57+
);
5258
let Some(profile) = policy.session_profiles.get(&file.session_profile) else {
5359
return Err(problems);
5460
};
61+
let profiled: BTreeSet<&str> = profile
62+
.snowflake
63+
.keys()
64+
.chain(profile.pinned.keys())
65+
.map(String::as_str)
66+
.collect();
67+
problems.extend(file.cases.iter().flat_map(|case| {
68+
profile_session_problems(&format!("{relative}#{}", case.id), case, &profiled)
69+
}));
5570
if !problems.is_empty() {
5671
return Err(problems);
5772
}
@@ -103,14 +118,15 @@ fn expand(
103118
) -> LoadedCase {
104119
let sql = source_sql(case, spelling);
105120
let setup_sql: Vec<String> = case.functions.iter().map(create_function).collect();
121+
let hashed_setup: Vec<String> = setup_sql.iter().chain(&case.setup).cloned().collect();
106122
let values = case.values.unwrap_or(true);
107123
let hash = case_hash(&HashedContent {
108124
format: EVIDENCE_FORMAT,
109125
source_dialect: SOURCE_DIALECT,
110126
session_parameters: parameters,
111127
source_sql: &sql,
112128
values: (!values).then_some(false),
113-
setup_sql: &setup_sql,
129+
setup_sql: &hashed_setup,
114130
});
115131
LoadedCase {
116132
reference: String::new(),
@@ -131,6 +147,7 @@ fn expand(
131147
functions: case.functions.clone(),
132148
overrides: case.overrides.clone(),
133149
setup_sql,
150+
setup: case.setup.clone(),
134151
inputs: inputs(case),
135152
row_sources: row_sources(case, spelling),
136153
}

‎crates/sqlquack-evidence/src/cases/_helpers/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ pub(crate) mod files;
22
pub(crate) mod functions;
33
pub(crate) mod hashing;
44
pub(crate) mod loading;
5+
pub(crate) mod setup;
56
pub(crate) mod source_sql;

0 commit comments

Comments
 (0)