From b9df2c0fe87f2ca88aa449d22ee17b164e9eacfe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:10:10 +0000 Subject: [PATCH 1/6] test(benchmarks): add the DuckShop edge-case area with semi-structured cases Add a separate edge-case area to DuckShop: models under models//edge_cases/ tagged edge_case, their own raw source and fixed-row loader that data_scale does not scale, and unit tests whose expected rows were captured on Snowflake under a pinned session. The first batch covers semi-structured data: JSON null against SQL NULL and missing keys, document text with sorted, case-differing and escaped keys, JSON number fidelity and casts out of VARIANT, path-like and computed keys with GET and GET_IGNORE_CASE, object builders and editors (OBJECT_CONSTRUCT_KEEP_NULL, OBJECT_INSERT, OBJECT_DELETE, OBJECT_PICK), array builders, ARRAY_EXCEPT and number spellings in ARRAY_POSITION, FLATTEN OUTER over empty, null and object inputs, SPLIT_TO_TABLE, lenient and malformed text in TRY_PARSE_JSON, subscripts, typed lambdas and casts of parsed fields as grouping and window keys. A custom rule, XSQBRDUCK002, keeps the edge_cases folders and the edge_case tag in step. The edge connection pins the session the expectations were captured under, and the project no longer sets QUOTED_IDENTIFIERS_IGNORE_CASE. --- .../edge_cases/int_edge_payload_arrays.sql | 56 ++ .../int_edge_payload_attributes.sql | 34 + .../edge_cases/int_edge_payload_documents.sql | 69 ++ .../int_edge_payload_item_totals.sql | 44 ++ .../edge_cases/int_edge_payload_lines.sql | 36 + .../edge_cases/int_edge_payload_objects.sql | 83 +++ .../edge_cases/int_edge_payload_tags.sql | 33 + .../edge_cases/int_edge_text_payload_skus.sql | 54 ++ .../edge_cases/fct_edge_payload_events.sql | 53 ++ .../rpt_edge_partner_payload_fields.sql | 36 + .../edge_cases/stg_edge_partner_payloads.sql | 78 +++ .../project/python/loaders/_edge_cases.py | 95 +++ .../project/python/loaders/edge_cases.py | 23 + .../duckshop/project/rules/area_tags.py | 36 + .../duckshop/project/sources/edge_cases.yml | 34 + .../duckshop/project/sqlbuild_project.toml | 42 +- .../project/tests/test_rule_area_tags.py | 66 ++ .../test_int_edge_payload_arrays.sql | 122 ++++ .../test_int_edge_payload_attributes.sql | 95 +++ .../test_int_edge_payload_documents.sql | 232 +++++++ .../test_int_edge_payload_item_totals.sql | 56 ++ .../test_int_edge_payload_lines.sql | 101 +++ .../test_int_edge_payload_objects.sql | 90 +++ .../edge_cases/test_int_edge_payload_tags.sql | 56 ++ .../test_int_edge_text_payload_skus.sql | 53 ++ .../test_fct_edge_payload_events.sql | 41 ++ .../test_rpt_edge_partner_payload_fields.sql | 44 ++ .../test_stg_edge_partner_payloads.sql | 621 ++++++++++++++++++ 28 files changed, 2379 insertions(+), 4 deletions(-) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_attributes.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_documents.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_item_totals.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_lines.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_tags.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_text_payload_skus.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_payload_events.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_payload_fields.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_payloads.sql create mode 100644 benchmarks/duckshop/project/python/loaders/_edge_cases.py create mode 100644 benchmarks/duckshop/project/python/loaders/edge_cases.py create mode 100644 benchmarks/duckshop/project/rules/area_tags.py create mode 100644 benchmarks/duckshop/project/sources/edge_cases.yml create mode 100644 benchmarks/duckshop/project/tests/test_rule_area_tags.py create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_attributes.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_documents.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_item_totals.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_lines.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_objects.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_tags.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_text_payload_skus.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_payload_events.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_payload_fields.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_payloads.sql diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql new file mode 100644 index 0000000000..9382ce1707 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql @@ -0,0 +1,56 @@ +MODEL ( + description """ + One row per partner payload message with its promotion codes and tags combined as arrays, and + category eligibility checked against ids the partner sent in mixed number spellings + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + codes (type ARRAY), + codes_compact (type ARRAY), + tags_compact (type ARRAY), + tags_kept (type ARRAY), + tags_all (type ARRAY), + tags_distinct (type ARRAY), + category_position (type INT), + is_eligible (type BOOLEAN), + shares_extra_tags (type BOOLEAN), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + ARRAY_CONSTRUCT( + p.payload:promo_code::VARCHAR, + p.payload:gift_code::VARCHAR + )::ARRAY AS codes, + ARRAY_CONSTRUCT_COMPACT( + p.payload:promo_code::VARCHAR, + p.payload:gift_code::VARCHAR + )::ARRAY AS codes_compact, + ARRAY_COMPACT(p.payload:tags::ARRAY)::ARRAY AS tags_compact, + ARRAY_EXCEPT(p.payload:tags::ARRAY, p.payload:removed_tags::ARRAY)::ARRAY AS tags_kept, + ARRAY_CAT( + p.payload:tags::ARRAY, + p.payload:extra_tags::ARRAY + )::ARRAY AS tags_all, + ARRAY_DISTINCT(p.payload:tags::ARRAY)::ARRAY AS tags_distinct, + ARRAY_POSITION(p.payload:category_id, p.payload:eligible_ids::ARRAY)::INT AS category_position, + ARRAY_CONTAINS(p.payload:category_id, p.payload:eligible_ids::ARRAY)::BOOLEAN AS is_eligible, + ARRAYS_OVERLAP( + p.payload:tags::ARRAY, + p.payload:extra_tags::ARRAY + )::BOOLEAN AS shares_extra_tags + FROM payloads AS p +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_attributes.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_attributes.sql new file mode 100644 index 0000000000..2c7058f7d2 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_attributes.sql @@ -0,0 +1,34 @@ +MODEL ( + description """ + One row per partner payload message and attribute name the partner asked for, with the attribute + read by a computed key, exactly and ignoring case + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null]), + wanted_index (type INT, nullable false, audits [not_null]), + attribute_name (type VARCHAR), + attribute_value (type VARCHAR), + attribute_value_any_case (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + w.index::INT AS wanted_index, + w.value::VARCHAR AS attribute_name, + GET(p.payload:attrs, w.value::VARCHAR)::VARCHAR AS attribute_value, + GET_IGNORE_CASE(p.payload:attrs, w.value::VARCHAR)::VARCHAR AS attribute_value_any_case + FROM payloads AS p, LATERAL FLATTEN(INPUT => p.payload:wanted) AS w +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_documents.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_documents.sql new file mode 100644 index 0000000000..fb0e7db7d0 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_documents.sql @@ -0,0 +1,69 @@ +MODEL ( + description """ + One row per partner payload message with the payload written back as text for change detection, + the colour field read as a JSON null, a missing key or a value, and merchant-defined attribute + names read with literal keys + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + payload_json (type VARCHAR), + payload_text_cast (type VARCHAR), + payload_md5 (type VARCHAR), + text_payload_json (type VARCHAR), + text_payload_kind (type VARCHAR), + colour_is_json_null (type BOOLEAN), + colour_kind (type VARCHAR), + colour_text (type VARCHAR), + colour_json (type VARCHAR), + colour_is_missing (type BOOLEAN), + colour_or_default (type VARCHAR), + size_cm (type VARCHAR), + dotted_attribute (type VARCHAR), + discount_attribute (type VARCHAR), + empty_attribute (type VARCHAR), + star_attribute (type VARCHAR), + path_like_attribute (type VARCHAR), + sku_any_case (type VARCHAR), + tag_list (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload, + s.text_payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + TO_JSON(p.payload)::VARCHAR AS payload_json, + p.payload::VARCHAR AS payload_text_cast, + MD5(TO_VARCHAR(p.payload))::VARCHAR AS payload_md5, + TO_JSON(p.text_payload)::VARCHAR AS text_payload_json, + TYPEOF(p.text_payload)::VARCHAR AS text_payload_kind, + IS_NULL_VALUE(p.payload:colour)::BOOLEAN AS colour_is_json_null, + TYPEOF(p.payload:colour)::VARCHAR AS colour_kind, + p.payload:colour::VARCHAR AS colour_text, + TO_JSON(p.payload:colour)::VARCHAR AS colour_json, + p.payload:colour IS NULL AS colour_is_missing, + COALESCE(p.payload:colour::VARCHAR, 'undyed')::VARCHAR AS colour_or_default, + GET(p.payload:attrs, 'Size (cm)')::VARCHAR AS size_cm, + GET(p.payload:attrs, 'a.b')::VARCHAR AS dotted_attribute, + p.payload:attrs['$discount']::VARCHAR AS discount_attribute, + GET(p.payload:attrs, '')::VARCHAR AS empty_attribute, + GET(p.payload:attrs, '*')::VARCHAR AS star_attribute, + GET(p.payload:attrs, '$.a')::VARCHAR AS path_like_attribute, + GET_IGNORE_CASE(p.payload:attrs, 'SKU')::VARCHAR AS sku_any_case, + ARRAY_TO_STRING( + ARRAY_COMPACT(p.payload:tags::ARRAY), + '|' + )::VARCHAR AS tag_list + FROM payloads AS p +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_item_totals.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_item_totals.sql new file mode 100644 index 0000000000..e6893cc0a4 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_item_totals.sql @@ -0,0 +1,44 @@ +MODEL ( + description """ + One row per partner payload message with its item lines totalled and filtered through lambdas, + including a lambda parameter named like a column of the model + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + sku (type VARCHAR), + item_total (type NUMBER(18, 2)), + priced_items (type ARRAY), + item_skus (type ARRAY), + priced_item_count (type INT), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + p.payload:sku::VARCHAR AS sku, + REDUCE( + p.payload:items, + 0::NUMBER(18, 2), + (acc NUMBER(18, 2), i VARIANT) -> acc + + COALESCE(i:qty::NUMBER(18, 2) * i:price::NUMBER(18, 2), 0) + )::NUMBER(18, 2) AS item_total, + FILTER(p.payload:items, sku -> sku:price IS NOT NULL)::ARRAY AS priced_items, + TRANSFORM( + p.payload:items, + i -> i:sku::VARCHAR + )::ARRAY AS item_skus, + ARRAY_SIZE(FILTER(p.payload:items, sku -> sku:price IS NOT NULL))::INT AS priced_item_count + FROM payloads AS p +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_lines.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_lines.sql new file mode 100644 index 0000000000..0158150f08 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_lines.sql @@ -0,0 +1,36 @@ +MODEL ( + description """ + One row per partner payload line, keeping messages whose lines are empty, a JSON null, missing + or a single object as one row without a line + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null]), + line_index (type INT), + line_key (type VARCHAR), + line_path (type VARCHAR), + line_kind (type VARCHAR), + line_sku (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + l.index::INT AS line_index, + l.key::VARCHAR AS line_key, + l.path::VARCHAR AS line_path, + TYPEOF(l.value)::VARCHAR AS line_kind, + l.value:sku::VARCHAR AS line_sku + FROM payloads AS p, LATERAL FLATTEN(INPUT => p.payload:lines, OUTER => TRUE) AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql new file mode 100644 index 0000000000..c9735db890 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql @@ -0,0 +1,83 @@ +MODEL ( + description """ + One row per partner payload message with order summaries built as objects that drop or keep + nulls, and the payload normalised to the current version by inserting, deleting and picking keys + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + order_summary (type OBJECT), + order_summary_keep_null (type OBJECT), + currency_object (type OBJECT), + normalised_payload (type OBJECT), + gift_note_updated (type OBJECT), + picked_payload (type OBJECT), + picked_by_array (type OBJECT), + picked_by_variant (type OBJECT), + payload_keys (type ARRAY), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +normalised AS ( + SELECT + p.message_id, + p.payload, + OBJECT_INSERT( + OBJECT_DELETE( + p.payload::OBJECT, + 'meta', + 'a.b' + ), + 'orderId', + p.payload:meta:order_id, + TRUE + )::OBJECT AS normalised_payload + FROM payloads AS p +), + +final AS ( + SELECT + n.message_id, + OBJECT_CONSTRUCT( + 'channel', + n.payload:channel::VARCHAR, + 'sku', + n.payload:sku::VARCHAR, + 'qty', + n.payload:qty + )::OBJECT AS order_summary, + OBJECT_CONSTRUCT_KEEP_NULL( + 'channel', + n.payload:channel::VARCHAR, + 'sku', + n.payload:sku::VARCHAR, + 'qty', + n.payload:qty + )::OBJECT AS order_summary_keep_null, + {'currency': n.payload:currency::VARCHAR}::OBJECT AS currency_object, + n.normalised_payload, + OBJECT_INSERT( + n.normalised_payload, + 'gift_note', + n.payload:gift_note::VARCHAR, + TRUE + )::OBJECT AS gift_note_updated, + OBJECT_PICK(n.normalised_payload, 'orderId', 'lines')::OBJECT AS picked_payload, + OBJECT_PICK( + n.payload::OBJECT, + n.payload:pick_keys::ARRAY + )::OBJECT AS picked_by_array, + OBJECT_PICK(n.payload::OBJECT, n.payload:pick_keys)::OBJECT AS picked_by_variant, + OBJECT_KEYS(n.payload::OBJECT)::ARRAY AS payload_keys + FROM normalised AS n +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_tags.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_tags.sql new file mode 100644 index 0000000000..8cac2623f5 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_tags.sql @@ -0,0 +1,33 @@ +MODEL ( + description """ + One row per tag in the comma-separated tag text a partner sends, including empty tags between + commas + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null]), + tag_index (type INT), + tag (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + p.message_id, + t.index::INT AS tag_index, + t.value::VARCHAR AS tag + FROM payloads AS p, LATERAL SPLIT_TO_TABLE( + p.payload:tags_text::VARCHAR, + ',' + ) AS t +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_text_payload_skus.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_text_payload_skus.sql new file mode 100644 index 0000000000..1077e8bb2f --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_text_payload_skus.sql @@ -0,0 +1,54 @@ +MODEL ( + description """ + One row per SKU named in partner message text, grouping and ranking by casts of fields parsed + from the text inside the grouping and window keys themselves + """, + tags [edge_case], + columns ( + sku (type VARCHAR), + messages (type INT, audits [not_null]), + carts (type INT), + latest_message_id (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.received_at, + s.payload_text + FROM __ref("stg_edge_partner_payloads") AS s +), + +parsed AS ( + SELECT + p.message_id, + p.received_at, + TRY_PARSE_JSON(p.payload_text) AS document + FROM payloads AS p +), + +ranked AS ( + SELECT + d.message_id, + d.document, + ROW_NUMBER() OVER ( + PARTITION BY d.document:sku::VARCHAR + ORDER BY d.received_at DESC, d.message_id DESC + ) AS recency + FROM parsed AS d +), + +final AS ( + SELECT + r.document:sku::VARCHAR AS sku, + COUNT(*)::INT AS messages, + COUNT(DISTINCT r.document:cart_id::VARCHAR)::INT AS carts, + MAX(IFF(r.recency = 1, r.message_id, NULL))::VARCHAR AS latest_message_id + FROM ranked AS r + GROUP BY + r.document:sku::VARCHAR +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_payload_events.sql b/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_payload_events.sql new file mode 100644 index 0000000000..e16bd4d6cf --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_payload_events.sql @@ -0,0 +1,53 @@ +MODEL ( + description """ + One row per partner payload message, published with an event document built from typed values: a + nullable channel, a FLOAT unit price, an exact amount and the placement date + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + partner_code (type VARCHAR), + event_document (type VARIANT), + event_document_text (type VARCHAR), + ), +); + +WITH payloads AS ( + SELECT + s.message_id, + s.partner_code, + s.payload, + s.amount_float, + s.amount, + s.placed_on + FROM __ref("stg_edge_partner_payloads") AS s +), + +documents AS ( + SELECT + p.message_id, + p.partner_code, + OBJECT_CONSTRUCT_KEEP_NULL( + 'channel', + p.payload:channel::VARCHAR, + 'unit_price', + p.amount_float, + 'amount', + p.amount, + 'placed_on', + p.placed_on + )::VARIANT AS event_document + FROM payloads AS p +), + +final AS ( + SELECT + d.message_id, + d.partner_code, + d.event_document, + d.event_document::VARCHAR AS event_document_text + FROM documents AS d +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_payload_fields.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_payload_fields.sql new file mode 100644 index 0000000000..e2509d4f65 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_payload_fields.sql @@ -0,0 +1,36 @@ +MODEL ( + description """ + One row per partner with how many of its payload messages carry a colour, counted on the JSON + path, on JSON nulls and on the colour read as text + """, + tags [edge_case], + columns ( + partner_code (type VARCHAR, nullable false, audits [not_null, unique]), + messages (type INT), + colour_values (type INT), + colour_json_nulls (type INT), + colour_texts (type INT), + ), +); + +WITH payloads AS ( + SELECT + s.partner_code, + s.payload + FROM __ref("stg_edge_partner_payloads") AS s +), + +final AS ( + SELECT + COALESCE(p.partner_code, 'UNKNOWN')::VARCHAR AS partner_code, + COUNT(*)::INT AS messages, + COUNT(p.payload:colour)::INT AS colour_values, + COUNT_IF(IS_NULL_VALUE(p.payload:colour))::INT AS colour_json_nulls, + COUNT(p.payload:colour::VARCHAR)::INT AS colour_texts + FROM payloads AS p + GROUP BY + 1 +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_payloads.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_payloads.sql new file mode 100644 index 0000000000..7452ba3d8b --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_payloads.sql @@ -0,0 +1,78 @@ +MODEL ( + description """ + One row per partner payload message, from its latest delivery, with payload fields read through + paths, subscripts and typed casts, and the raw message text parsed leniently + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + partner_code (type VARCHAR), + received_at (type TIMESTAMP_NTZ), + payload (type VARIANT), + payload_text (type VARCHAR), + text_payload (type VARIANT), + qty_kind (type VARCHAR), + qty_is_integer (type BOOLEAN), + qty_is_decimal (type BOOLEAN), + qty_is_double (type BOOLEAN), + qty_units (type NUMBER(18, 0)), + qty_text_units (type NUMBER(18, 2)), + amount (type NUMBER(18, 2)), + amount_float (type FLOAT), + amount_text (type VARCHAR), + transaction_id_text (type VARCHAR), + transaction_id (type NUMBER(38, 0)), + is_gift (type BOOLEAN), + placed_on (type DATE), + first_item_sku (type VARCHAR), + last_item_sku (type VARCHAR), + item_count (type INT), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__partner_payloads") +), + +latest AS ( + SELECT + s.message_id, + UPPER(TRIM(s.partner_code))::VARCHAR AS partner_code, + s.received_at, + s.payload, + s.payload_text + FROM src AS s + QUALIFY + ROW_NUMBER() OVER (PARTITION BY s.message_id ORDER BY s._loaded_at DESC, s.delivery_id DESC) = 1 +), + +final AS ( + SELECT + l.message_id, + l.partner_code, + l.received_at, + l.payload, + l.payload_text, + TRY_PARSE_JSON(l.payload_text) AS text_payload, + TYPEOF(l.payload:qty)::VARCHAR AS qty_kind, + IS_INTEGER(l.payload:qty)::BOOLEAN AS qty_is_integer, + IS_DECIMAL(l.payload:qty)::BOOLEAN AS qty_is_decimal, + IS_DOUBLE(l.payload:qty)::BOOLEAN AS qty_is_double, + l.payload:qty::NUMBER(18, 0) AS qty_units, + TRY_TO_NUMBER(l.payload:qty::VARCHAR, 18, 2)::NUMBER(18, 2) AS qty_text_units, + l.payload:amount::NUMBER(18, 2) AS amount, + l.payload:amount::FLOAT AS amount_float, + l.payload:amount::VARCHAR AS amount_text, + l.payload:txn_id::VARCHAR AS transaction_id_text, + l.payload:txn_id::NUMBER(38, 0) AS transaction_id, + l.payload:gift::BOOLEAN AS is_gift, + l.payload:placed_on::DATE AS placed_on, + l.payload:items[0]:sku::VARCHAR AS first_item_sku, + l.payload:items[ARRAY_SIZE(l.payload:items) - 1]:sku::VARCHAR AS last_item_sku, + ARRAY_SIZE(l.payload:items)::INT AS item_count + FROM latest AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/python/loaders/_edge_cases.py b/benchmarks/duckshop/project/python/loaders/_edge_cases.py new file mode 100644 index 0000000000..bc30f42dd7 --- /dev/null +++ b/benchmarks/duckshop/project/python/loaders/_edge_cases.py @@ -0,0 +1,95 @@ +"""Fixed rows for the edge-case area's raw tables. + +Each row carries edge values that the edge models and their unit tests exercise. JSON documents are +written as raw text, exactly as a partner would send them, so key order, key case, number spelling +and escapes reach Snowflake unchanged. The rows never depend on `data_scale`. +""" + +from __future__ import annotations + +from datetime import datetime + +Rows = list[dict[str, object]] + + +def _at(text: str) -> datetime: + return datetime.fromisoformat(text) + + +def partner_payloads() -> Rows: + documents = [ + ( + "PM-1001", + "mkt_eu", + '{"sku": "MALLARD-01", "qty": 2, "amount": 19.990000000000002, ' + '"txn_id": 12345678901234567890, "gift": "yes", "placed_on": "2026-03-01", ' + '"colour": null, "channel": "web", "currency": "GBP", ' + '"items": [{"sku": "MALLARD-01", "qty": 2, "price": 4.995}, {"QTY": 2, "sku": "TEAL-02"}], ' + '"attrs": {"Size (cm)": "12", "a.b": "dotted", "a": {"b": "nested"}, ' + '"$discount": "5%", "": "blank", "*": "star", "$.a": "dollar path", "Sku": "MALLARD-01"}, ' + '"meta": {"order_id": "DS-O-1001"}, "tags": ["bath", "gift", "bath", null], ' + '"removed_tags": ["bath"], "extra_tags": ["gift"], "promo_code": "SPRING", ' + '"gift_code": null, "category_id": 2, "eligible_ids": [1, 2.0, 3], ' + '"pick_keys": ["sku", "qty"], "wanted": ["Size (cm)", "sku", "SKU", "a.b", ""], ' + '"lines": [{"sku": "MALLARD-01"}, {"sku": "TEAL-02"}], "tags_text": "bath,,gift"}', + '{"sku": "MALLARD-01", "sku": "TEAL-02"}', + ), + ( + "PM-1002", + "MKT_EU", + '{"qty": "2.5", "amount": 1E+3, "txn_id": 9007199254740993, "gift": true, ' + '"colour": "teal", "channel": null, "currency": null, "items": [], ' + '"attrs": {"SKU": "TEAL-02", "sku": "teal-02"}, "tags": [], "removed_tags": [], ' + '"extra_tags": [], "category_id": "1", "eligible_ids": [1, 2.0, 3], "pick_keys": [], ' + '"wanted": [], "lines": [], "tags_text": ""}', + "{'sku': 'TEAL-02', 'cart_id': 'CART-7'}", + ), + ( + "PM-1003", + "giftbox", + '{"qty": 2e0, "amount": -0, "txn_id": "00042", "gift": "N", "placed_on": null, ' + '"items": [{"sku": "EIDER-03"}], "attrs": "not an object", ' + '"tags": [1, 1.0, "1", {"b": 1, "a": 2}], "removed_tags": [1.0], "extra_tags": [1e0], ' + '"category_id": 3, "eligible_ids": [1, 2.0, 3e0], "lines": null, ' + '"note": "unit\\u001fseparator", "line": "para\\u2028graph"}', + '"{\\"orderId\\": \\"DS-O-1003\\"}"', + ), + ( + "PM-1004", + "giftbox", + '{"qty": null, "amount": 1e-7, "gift": 0, ' + '"lines": {"sku": "PINTAIL-04", "qty": 1}, "\\u00e9": "precomposed", ' + '"e\\u0301": "combining", "tags_text": null}', + '{"weight_kg": NaN, "lines": [1, 2,]}', + ), + ] + rows: Rows = [] + for index, (message_id, partner_code, payload, payload_text) in enumerate(documents): + rows.append( + { + "delivery_id": f"DL-{index + 1:04d}", + "message_id": message_id, + "partner_code": partner_code, + "received_at": _at("2026-03-01 09:00:00"), + "payload": payload, + "payload_text": payload_text, + "_loaded_at": _at("2026-03-01 10:00:00"), + } + ) + rows.append( + { + "delivery_id": "DL-0005", + "message_id": "PM-1001", + "partner_code": "mkt_eu", + "received_at": _at("2026-03-01 08:00:00"), + "payload": '{"sku": "MALLARD-01", "qty": 1}', + "payload_text": None, + "_loaded_at": _at("2026-03-01 09:30:00"), + } + ) + return rows + + +TABLES: dict[str, Rows] = { + "edge__partner_payloads": partner_payloads(), +} diff --git a/benchmarks/duckshop/project/python/loaders/edge_cases.py b/benchmarks/duckshop/project/python/loaders/edge_cases.py new file mode 100644 index 0000000000..fe5a743530 --- /dev/null +++ b/benchmarks/duckshop/project/python/loaders/edge_cases.py @@ -0,0 +1,23 @@ +"""Loaders for the edge-case area's raw tables, with the fixed rows in python.loaders._edge_cases. + +These rows are hand-picked edge values, not generated volume: `data_scale` does not change them, +and they never touch the core tables, so core outputs and expectations stay the same. +""" + +from python.loaders._edge_cases import TABLES +from python.loaders._writer import statements + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader + + +def _load(ctx: LoaderContext, source_name: str) -> object: + rows = TABLES[source_name] + for sql in statements(ctx.destination, source_name, rows): + ctx.execute_sql(sql) + return ctx.result(metadata={"rows": len(rows)}) + + +@loader +def edge__partner_payloads(ctx: LoaderContext) -> object: + return _load(ctx, "edge__partner_payloads") diff --git a/benchmarks/duckshop/project/rules/area_tags.py b/benchmarks/duckshop/project/rules/area_tags.py new file mode 100644 index 0000000000..bdb10fe45c --- /dev/null +++ b/benchmarks/duckshop/project/rules/area_tags.py @@ -0,0 +1,36 @@ +"""DuckShop house rule that keeps the edge-case area selectable on its own.""" + +import re + +from sqlbuild.rules import Finding, Model, RuleContext, rule + +EDGE_FOLDER = "edge_cases" +EDGE_TAG = "edge_case" +_HEADER_TAGS = re.compile(r"^[ \t]*tags\s*\[([^\]]*)\]", re.MULTILINE) + + +def _header_tags(sql: str) -> tuple[set[str], int]: + header_end = sql.find(");") + match = _HEADER_TAGS.search(sql, 0, header_end if header_end >= 0 else len(sql)) + if match is None: + return set(), 1 + tags = {tag.strip().strip("\"'") for tag in match.group(1).split(",") if tag.strip()} + return tags, sql[: match.start()].count("\n") + 1 + + +@rule( + code="XSQBRDUCK002", + message="Models in an edge_cases folder, and only those, must declare the edge_case tag", + remediation=( + "Add `tags [edge_case]` to the header of every model under models//edge_cases/, " + "and move a model that declares the tag into that folder, so benchmark and core runs can " + "exclude the edge-case area with --exclude tag:edge_case." + ), +) +def edge_area_is_tagged(*, model: Model, ctx: RuleContext) -> list[Finding]: + parts = ctx.project.tree.relative_parts(path=model.path, under="models") + in_area = EDGE_FOLDER in parts + tags, line = _header_tags(ctx.sql.for_model(model).authored.source) + if in_area == (EDGE_TAG in tags): + return [] + return [ctx.finding(subject=model, line=line)] diff --git a/benchmarks/duckshop/project/sources/edge_cases.yml b/benchmarks/duckshop/project/sources/edge_cases.yml new file mode 100644 index 0000000000..6f68757c46 --- /dev/null +++ b/benchmarks/duckshop/project/sources/edge_cases.yml @@ -0,0 +1,34 @@ +# Edge-case area: raw partner payload feed with hand-picked JSON edge values, written by the fixed +# loaders in python/loaders/edge_cases.py. Not scaled by data_scale. +sources: + - name: edge__partner_payloads + description: >- + Append-only partner order payloads holding JSON edge values, with the raw message text + beside each parsed payload + managed: true + contract: enforced + columns: + - name: delivery_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: message_id + type: VARCHAR + nullable: false + audits: + - not_null + - name: partner_code + type: VARCHAR + - name: received_at + type: TIMESTAMP_NTZ + - name: payload + type: VARIANT + - name: payload_text + type: VARCHAR + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 2c18525d70..5ce062fd4f 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -9,16 +9,50 @@ account = "example-account" user = "example_user" role = "EXAMPLE_ROLE" warehouse = "EXAMPLE_WH" -# SQLBuild 0.175.0 binds inspected Snowflake source columns as quoted lower-case names, which -# unquoted references cannot match, so planning fails once the raw tables exist. Ignoring the case -# of quoted identifiers avoids that; DuckShop's SQL quotes no identifiers whose case matters. -session_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true } + +# The edge-case area's expected values are captured on Snowflake under this pinned session: UTC, +# with every parameter that changes a result or how text is parsed set to Snowflake's default. +# Run the edge area on this connection only (see RUNBOOK.md, "Edge cases"). +[connections.edge] +account = "example-account" +user = "example_user" +role = "EXAMPLE_ROLE" +warehouse = "EXAMPLE_WH" + +[connections.edge.session_parameters] +TIMEZONE = "UTC" +TIMESTAMP_TYPE_MAPPING = "TIMESTAMP_NTZ" +WEEK_START = 0 +WEEK_OF_YEAR_POLICY = 0 +DEFAULT_NULL_ORDERING = "LAST" +QUOTED_IDENTIFIERS_IGNORE_CASE = false +USE_CACHED_RESULT = false +DATE_INPUT_FORMAT = "AUTO" +DATE_OUTPUT_FORMAT = "YYYY-MM-DD" +TIME_INPUT_FORMAT = "AUTO" +TIME_OUTPUT_FORMAT = "HH24:MI:SS" +TIMESTAMP_INPUT_FORMAT = "AUTO" +TIMESTAMP_OUTPUT_FORMAT = "YYYY-MM-DD HH24:MI:SS.FF3 TZHTZM" +TIMESTAMP_NTZ_OUTPUT_FORMAT = "YYYY-MM-DD HH24:MI:SS.FF3" +TIMESTAMP_LTZ_OUTPUT_FORMAT = "" +TIMESTAMP_TZ_OUTPUT_FORMAT = "" +TWO_DIGIT_CENTURY_START = 1970 +BINARY_INPUT_FORMAT = "HEX" +BINARY_OUTPUT_FORMAT = "HEX" +TIMESTAMP_DAY_IS_ALWAYS_24H = false +JSON_INDENT = 2 +STRICT_JSON_OUTPUT = false [targets.dev] connection = "warehouse" database = "DUCKSHOP" schema = "DEV" +[targets.edge] +connection = "edge" +database = "DUCKSHOP" +schema = "EDGE" + [targets.prod] connection = "warehouse" database = "DUCKSHOP" diff --git a/benchmarks/duckshop/project/tests/test_rule_area_tags.py b/benchmarks/duckshop/project/tests/test_rule_area_tags.py new file mode 100644 index 0000000000..cebc6af87e --- /dev/null +++ b/benchmarks/duckshop/project/tests/test_rule_area_tags.py @@ -0,0 +1,66 @@ +from rules.area_tags import edge_area_is_tagged + +from sqlbuild.rules import RuleCase, evaluate_rule + +_TAGGED = 'MODEL (\n description "Edge orders",\n tags [edge_case],\n);\nSELECT 1 AS order_id\n' +_UNTAGGED = 'MODEL (\n description "Orders",\n);\nSELECT 1 AS order_id\n' +_OTHER_TAG = ( + 'MODEL (\n description "Orders",\n tags [known_divergence],\n);\nSELECT 1 AS order_id\n' +) + + +def test_given_tagged_model_in_edge_folder_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=edge_area_is_tagged, + test_case=RuleCase( + description="edge model declares the edge_case tag", + source=_TAGGED, + path="models/intermediate/edge_cases/int_edge_orders.sql", + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 + + +def test_given_untagged_model_in_edge_folder_when_evaluating_then_reports_it() -> None: + result = evaluate_rule( + rule=edge_area_is_tagged, + test_case=RuleCase( + description="edge model without the edge_case tag", + source=_OTHER_TAG, + path="models/staging/edge_cases/stg_edge_orders.sql", + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + assert result.findings[0].line == 3 + + +def test_given_tagged_model_outside_edge_folder_when_evaluating_then_reports_it() -> None: + result = evaluate_rule( + rule=edge_area_is_tagged, + test_case=RuleCase( + description="core model declares the edge_case tag", + source=_TAGGED, + path="models/marts/orders/fct_orders.sql", + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + + +def test_given_untagged_model_outside_edge_folder_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=edge_area_is_tagged, + test_case=RuleCase( + description="core model without the tag", + source=_UNTAGGED, + path="models/marts/orders/fct_orders.sql", + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql new file mode 100644 index 0000000000..daaf0aa323 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql @@ -0,0 +1,122 @@ +TEST (name "int_edge_payload_arrays__combines_codes_and_tags_with_nulls_and_duplicates"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"promo_code": "SPRING", "gift_code": null, "tags": ["bath", "gift", "bath", null, ' + || '"bath"], "removed_tags": ["bath"], "extra_tags": ["gift", null]}' + ) AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"promo_code": null, "tags": [], "removed_tags": [], "extra_tags": []}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON( + '{"gift_code": "GIFT-10", "tags": [1, 1.0, "1", {"b": 1, "a": 2}, {"a": 2, "b": 1}], ' + || '"removed_tags": [1.0, {"a": 2, "b": 1}], "extra_tags": [1e0]}' + ) +), + +__expected__int_edge_payload_arrays AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('["SPRING",undefined]')::ARRAY AS codes, + PARSE_JSON('["SPRING"]')::ARRAY AS codes_compact, + PARSE_JSON('["bath","gift","bath","bath"]')::ARRAY AS tags_compact, + PARSE_JSON('["gift","bath",null,"bath"]')::ARRAY AS tags_kept, + PARSE_JSON('["bath","gift","bath",null,"bath","gift",null]')::ARRAY AS tags_all, + PARSE_JSON('["bath","gift",null]')::ARRAY AS tags_distinct, + TRUE AS shares_extra_tags + UNION ALL + SELECT + 'PM-2' AS message_id, + PARSE_JSON('[undefined,undefined]')::ARRAY AS codes, + PARSE_JSON('[]')::ARRAY AS codes_compact, + PARSE_JSON('[]')::ARRAY AS tags_compact, + PARSE_JSON('[]')::ARRAY AS tags_kept, + PARSE_JSON('[]')::ARRAY AS tags_all, + PARSE_JSON('[]')::ARRAY AS tags_distinct, + FALSE AS shares_extra_tags + UNION ALL + SELECT + 'PM-3' AS message_id, + PARSE_JSON('[undefined,"GIFT-10"]')::ARRAY AS codes, + PARSE_JSON('["GIFT-10"]')::ARRAY AS codes_compact, + PARSE_JSON('[1,1,"1",{"a":2,"b":1},{"a":2,"b":1}]')::ARRAY AS tags_compact, + PARSE_JSON('[1,"1",{"a":2,"b":1}]')::ARRAY AS tags_kept, + PARSE_JSON('[1,1,"1",{"a":2,"b":1},{"a":2,"b":1},1.000000000000000e+00]')::ARRAY AS tags_all, + PARSE_JSON('[1,"1",{"a":2,"b":1}]')::ARRAY AS tags_distinct, + TRUE AS shares_extra_tags +) + +TEST (name "int_edge_payload_arrays__matches_category_ids_across_number_spellings"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"category_id": 2, "eligible_ids": [1, 2.0, 3]}') AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"category_id": 3, "eligible_ids": [1, 2.0, 3e0]}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"category_id": "1", "eligible_ids": [1, 2.0, 3]}') + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{"category_id": 1.0, "eligible_ids": [1, 2.0, 3]}') + UNION ALL + SELECT + 'PM-5', + PARSE_JSON('{"category_id": 2.5, "eligible_ids": [2, 2.50, 3]}') + UNION ALL + SELECT + 'PM-6', + PARSE_JSON('{"category_id": null, "eligible_ids": [null, 1]}') + UNION ALL + SELECT + 'PM-7', + PARSE_JSON('{"category_id": 4, "eligible_ids": []}') +), + +__expected__int_edge_payload_arrays AS ( + SELECT + 'PM-1' AS message_id, + 1 AS category_position, + TRUE AS is_eligible + UNION ALL + SELECT + 'PM-2' AS message_id, + 2 AS category_position, + TRUE AS is_eligible + UNION ALL + SELECT + 'PM-3' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS category_position, + FALSE AS is_eligible + UNION ALL + SELECT + 'PM-4' AS message_id, + 0 AS category_position, + TRUE AS is_eligible + UNION ALL + SELECT + 'PM-5' AS message_id, + 1 AS category_position, + TRUE AS is_eligible + UNION ALL + SELECT + 'PM-6' AS message_id, + 0 AS category_position, + TRUE AS is_eligible + UNION ALL + SELECT + 'PM-7' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS category_position, + FALSE AS is_eligible +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_attributes.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_attributes.sql new file mode 100644 index 0000000000..ac90b686f7 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_attributes.sql @@ -0,0 +1,95 @@ +TEST (name "int_edge_payload_attributes__reads_computed_keys_exactly_and_ignoring_case"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"wanted": ["Size (cm)", "sku", "SKU", "a.b", "$discount", "", "*", "colour"], ' + || '"attrs": {"Size (cm)": "12", "Sku": "MALLARD-01", "a.b": "dotted", "a": {"b": "nested"}, ' + || '"$discount": "5%", "": "blank", "*": "star", "colour": null}}' + ) AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"wanted": ["sku", "SKU"], "attrs": {"SKU": "TEAL-02", "sku": "teal-02"}}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"wanted": [], "attrs": {"sku": "EIDER-03"}}') + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{"attrs": {"sku": "PINTAIL-04"}}') +), + +__expected__int_edge_payload_attributes AS ( + SELECT + 'PM-1' AS message_id, + 0 AS wanted_index, + 'Size (cm)' AS attribute_name, + '12' AS attribute_value, + '12' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 1 AS wanted_index, + 'sku' AS attribute_name, + CAST(NULL AS VARCHAR) AS attribute_value, + 'MALLARD-01' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 2 AS wanted_index, + 'SKU' AS attribute_name, + CAST(NULL AS VARCHAR) AS attribute_value, + 'MALLARD-01' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 3 AS wanted_index, + 'a.b' AS attribute_name, + 'dotted' AS attribute_value, + 'dotted' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 4 AS wanted_index, + '$discount' AS attribute_name, + '5%' AS attribute_value, + '5%' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 5 AS wanted_index, + '' AS attribute_name, + 'blank' AS attribute_value, + 'blank' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 6 AS wanted_index, + '*' AS attribute_name, + 'star' AS attribute_value, + 'star' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-1' AS message_id, + 7 AS wanted_index, + 'colour' AS attribute_name, + CAST(NULL AS VARCHAR) AS attribute_value, + CAST(NULL AS VARCHAR) AS attribute_value_any_case + UNION ALL + SELECT + 'PM-2' AS message_id, + 0 AS wanted_index, + 'sku' AS attribute_name, + 'teal-02' AS attribute_value, + 'teal-02' AS attribute_value_any_case + UNION ALL + SELECT + 'PM-2' AS message_id, + 1 AS wanted_index, + 'SKU' AS attribute_name, + 'TEAL-02' AS attribute_value, + 'TEAL-02' AS attribute_value_any_case +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_documents.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_documents.sql new file mode 100644 index 0000000000..1de77236c3 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_documents.sql @@ -0,0 +1,232 @@ +TEST (name "int_edge_payload_documents__writes_documents_with_sorted_keys_and_escapes"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"sku": "MALLARD-01", "qty": 2, "Attrs": {"sku": "a", "SKU": "b"}}') AS payload, + CAST(NULL AS VARIANT) AS text_payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"items": [{"qty": 1}, {"QTY": 2}], "b": 1, "a": {"z": 1, "y": 2}}'), + NULL + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"\\u00e9": "precomposed", "e\\u0301": "combining", "_": 1, "Z": 2, "a": 3}'), + NULL + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{"note": "unit\\u001fseparator", "line": "para\\u2028graph", "path": "a\\\\b \\"quoted\\" /"}'), + NULL +), + +__expected__int_edge_payload_documents AS ( + SELECT + 'PM-1' AS message_id, + '{"Attrs":{"SKU":"b","sku":"a"},"qty":2,"sku":"MALLARD-01"}' AS payload_json, + '{"Attrs":{"SKU":"b","sku":"a"},"qty":2,"sku":"MALLARD-01"}' AS payload_text_cast, + '1f7dde3ac20d13ae115fb2d5e225d698' AS payload_md5 + UNION ALL + SELECT + 'PM-2' AS message_id, + '{"a":{"y":2,"z":1},"b":1,"items":[{"qty":1},{"QTY":2}]}' AS payload_json, + '{"a":{"y":2,"z":1},"b":1,"items":[{"qty":1},{"QTY":2}]}' AS payload_text_cast, + '565e04373d88d13210a746e00e30f9a6' AS payload_md5 + UNION ALL + SELECT + 'PM-3' AS message_id, + '{"Z":2,"_":1,"a":3,"e\u0301":"combining","é":"precomposed"}' AS payload_json, + '{"Z":2,"_":1,"a":3,"e\u0301":"combining","é":"precomposed"}' AS payload_text_cast, + '40585679ec958a682a75265757a619d4' AS payload_md5 + UNION ALL + SELECT + 'PM-4' AS message_id, + '{"line":"para\u2028graph","note":"unit\\u001fseparator","path":"a\\\\b \\"quoted\\" /"}' AS payload_json, + '{"line":"para\u2028graph","note":"unit\\u001fseparator","path":"a\\\\b \\"quoted\\" /"}' AS payload_text_cast, + '903fbf2b725c8f036f83e967faece91d' AS payload_md5 +) + +TEST (name "int_edge_payload_documents__tells_json_null_from_missing_and_sql_null"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"colour": null}') AS payload, + CAST(NULL AS VARIANT) AS text_payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{}'), + NULL + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"colour": "teal"}'), + NULL + UNION ALL + SELECT + 'PM-4', + CAST(NULL AS VARIANT), + NULL + UNION ALL + SELECT + 'PM-5', + PARSE_JSON('{"colour": ""}'), + NULL + UNION ALL + SELECT + 'PM-6', + PARSE_JSON('{"colour": ["teal", null]}'), + NULL +), + +__expected__int_edge_payload_documents AS ( + SELECT + 'PM-1' AS message_id, + TRUE AS colour_is_json_null, + 'NULL_VALUE' AS colour_kind, + CAST(NULL AS VARCHAR) AS colour_text, + 'null' AS colour_json, + FALSE AS colour_is_missing, + 'undyed' AS colour_or_default + UNION ALL + SELECT + 'PM-2' AS message_id, + CAST(NULL AS BOOLEAN) AS colour_is_json_null, + CAST(NULL AS VARCHAR) AS colour_kind, + CAST(NULL AS VARCHAR) AS colour_text, + CAST(NULL AS VARCHAR) AS colour_json, + TRUE AS colour_is_missing, + 'undyed' AS colour_or_default + UNION ALL + SELECT + 'PM-3' AS message_id, + FALSE AS colour_is_json_null, + 'VARCHAR' AS colour_kind, + 'teal' AS colour_text, + '"teal"' AS colour_json, + FALSE AS colour_is_missing, + 'teal' AS colour_or_default + UNION ALL + SELECT + 'PM-4' AS message_id, + CAST(NULL AS BOOLEAN) AS colour_is_json_null, + CAST(NULL AS VARCHAR) AS colour_kind, + CAST(NULL AS VARCHAR) AS colour_text, + CAST(NULL AS VARCHAR) AS colour_json, + TRUE AS colour_is_missing, + 'undyed' AS colour_or_default + UNION ALL + SELECT + 'PM-5' AS message_id, + FALSE AS colour_is_json_null, + 'VARCHAR' AS colour_kind, + '' AS colour_text, + '""' AS colour_json, + FALSE AS colour_is_missing, + '' AS colour_or_default + UNION ALL + SELECT + 'PM-6' AS message_id, + FALSE AS colour_is_json_null, + 'ARRAY' AS colour_kind, + '["teal",null]' AS colour_text, + '["teal",null]' AS colour_json, + FALSE AS colour_is_missing, + '["teal",null]' AS colour_or_default +) + +TEST (name "int_edge_payload_documents__reads_attribute_names_that_look_like_paths"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"attrs": {"Size (cm)": "12", "a.b": "dotted", "a": {"b": "nested"}, "$discount": "5%", ' + || '"": "blank", "*": "star", "$.a": "dollar path", "Sku": "MALLARD-01"}}' + ) AS payload, + CAST(NULL AS VARIANT) AS text_payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"attrs": {"a": {"b": "nested only"}, "SKU": "TEAL-02", "sku": "teal-02"}}'), + NULL + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"attrs": "not an object"}'), + NULL +), + +__expected__int_edge_payload_documents AS ( + SELECT + 'PM-1' AS message_id, + '12' AS size_cm, + 'dotted' AS dotted_attribute, + '5%' AS discount_attribute, + 'blank' AS empty_attribute, + 'star' AS star_attribute, + 'dollar path' AS path_like_attribute, + 'MALLARD-01' AS sku_any_case + UNION ALL + SELECT + 'PM-2' AS message_id, + CAST(NULL AS VARCHAR) AS size_cm, + CAST(NULL AS VARCHAR) AS dotted_attribute, + CAST(NULL AS VARCHAR) AS discount_attribute, + CAST(NULL AS VARCHAR) AS empty_attribute, + CAST(NULL AS VARCHAR) AS star_attribute, + CAST(NULL AS VARCHAR) AS path_like_attribute, + 'TEAL-02' AS sku_any_case + UNION ALL + SELECT + 'PM-3' AS message_id, + CAST(NULL AS VARCHAR) AS size_cm, + CAST(NULL AS VARCHAR) AS dotted_attribute, + CAST(NULL AS VARCHAR) AS discount_attribute, + CAST(NULL AS VARCHAR) AS empty_attribute, + CAST(NULL AS VARCHAR) AS star_attribute, + CAST(NULL AS VARCHAR) AS path_like_attribute, + CAST(NULL AS VARCHAR) AS sku_any_case +) + +TEST (name "int_edge_payload_documents__writes_text_payloads_and_tag_lists"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"tags": ["bath", null, "say \\"hi\\"", "a\\\\b", 7, ""]}') AS payload, + PARSE_JSON('"{\\"orderId\\": \\"DS-O-1001\\"}"') AS text_payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"tags": []}'), + PARSE_JSON('null') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"tags": [1.50, true, {"b": 1, "a": 2}]}'), + PARSE_JSON('{"b": 2, "a": [1.50, 1e2, -0.0]}') +), + +__expected__int_edge_payload_documents AS ( + SELECT + 'PM-1' AS message_id, + 'bath|say "hi"|a\\b|7|' AS tag_list, + '"{\\"orderId\\": \\"DS-O-1001\\"}"' AS text_payload_json, + 'VARCHAR' AS text_payload_kind + UNION ALL + SELECT + 'PM-2' AS message_id, + '' AS tag_list, + 'null' AS text_payload_json, + 'NULL_VALUE' AS text_payload_kind + UNION ALL + SELECT + 'PM-3' AS message_id, + '1.5|true|{"a":2,"b":1}' AS tag_list, + '{"a":[1.5,1.000000000000000e+02,0],"b":2}' AS text_payload_json, + 'OBJECT' AS text_payload_kind +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_item_totals.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_item_totals.sql new file mode 100644 index 0000000000..56d304d756 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_item_totals.sql @@ -0,0 +1,56 @@ +TEST (name "int_edge_payload_item_totals__totals_and_filters_items_through_typed_lambdas"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"sku": "BUNDLE-01", "items": [{"sku": "MALLARD-01", "qty": 2, "price": 4.995}, ' + || '{"sku": "TEAL-02", "qty": 1, "price": "3.10"}, {"sku": "EIDER-03", "qty": 3}]}' + ) AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"sku": "BUNDLE-02", "items": []}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"items": [null, {"sku": null, "qty": 1, "price": 0.1}]}') + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{"sku": "MALLARD-01"}') +), + +__expected__int_edge_payload_item_totals AS ( + SELECT + 'PM-1' AS message_id, + 'BUNDLE-01' AS sku, + 13.10::NUMBER(18, 2) AS item_total, + PARSE_JSON('[{"price":4.995,"qty":2,"sku":"MALLARD-01"},{"price":"3.10","qty":1,"sku":"TEAL-02"}]')::ARRAY AS priced_items, + PARSE_JSON('["MALLARD-01","TEAL-02","EIDER-03"]')::ARRAY AS item_skus, + 2 AS priced_item_count + UNION ALL + SELECT + 'PM-2' AS message_id, + 'BUNDLE-02' AS sku, + 0.00::NUMBER(18, 2) AS item_total, + PARSE_JSON('[]')::ARRAY AS priced_items, + PARSE_JSON('[]')::ARRAY AS item_skus, + 0 AS priced_item_count + UNION ALL + SELECT + 'PM-3' AS message_id, + CAST(NULL AS VARCHAR) AS sku, + 0.10::NUMBER(18, 2) AS item_total, + PARSE_JSON('[{"price":0.1,"qty":1,"sku":null}]')::ARRAY AS priced_items, + PARSE_JSON('[undefined,undefined]')::ARRAY AS item_skus, + 1 AS priced_item_count + UNION ALL + SELECT + 'PM-4' AS message_id, + 'MALLARD-01' AS sku, + CAST(NULL AS NUMBER(18, 2)) AS item_total, + CAST(NULL AS ARRAY) AS priced_items, + CAST(NULL AS ARRAY) AS item_skus, + CAST(NULL AS NUMBER(38, 0)) AS priced_item_count +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_lines.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_lines.sql new file mode 100644 index 0000000000..22fc3c1f5e --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_lines.sql @@ -0,0 +1,101 @@ +TEST (name "int_edge_payload_lines__keeps_messages_without_line_arrays_as_one_row"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"lines": [{"sku": "MALLARD-01"}, {"sku": "TEAL-02"}]}') AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"lines": []}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"lines": null}') + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{}') + UNION ALL + SELECT + 'PM-5', + PARSE_JSON('{"lines": {"sku": "EIDER-03", "qty": 1}}') + UNION ALL + SELECT + 'PM-6', + PARSE_JSON('{"lines": [null, "PINTAIL-04"]}') +), + +__expected__int_edge_payload_lines AS ( + SELECT + 'PM-1' AS message_id, + 0 AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + '[0]' AS line_path, + 'OBJECT' AS line_kind, + 'MALLARD-01' AS line_sku + UNION ALL + SELECT + 'PM-1' AS message_id, + 1 AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + '[1]' AS line_path, + 'OBJECT' AS line_kind, + 'TEAL-02' AS line_sku + UNION ALL + SELECT + 'PM-2' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + '' AS line_path, + CAST(NULL AS VARCHAR) AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-3' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + CAST(NULL AS VARCHAR) AS line_path, + CAST(NULL AS VARCHAR) AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-4' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + CAST(NULL AS VARCHAR) AS line_path, + CAST(NULL AS VARCHAR) AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-5' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS line_index, + 'qty' AS line_key, + 'qty' AS line_path, + 'INTEGER' AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-5' AS message_id, + CAST(NULL AS NUMBER(38, 0)) AS line_index, + 'sku' AS line_key, + 'sku' AS line_path, + 'VARCHAR' AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-6' AS message_id, + 0 AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + '[0]' AS line_path, + 'NULL_VALUE' AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku + UNION ALL + SELECT + 'PM-6' AS message_id, + 1 AS line_index, + CAST(NULL AS VARCHAR) AS line_key, + '[1]' AS line_path, + 'VARCHAR' AS line_kind, + CAST(NULL AS VARCHAR) AS line_sku +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_objects.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_objects.sql new file mode 100644 index 0000000000..9f63953772 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_objects.sql @@ -0,0 +1,90 @@ +TEST (name "int_edge_payload_objects__builds_summaries_that_drop_or_keep_nulls"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"channel": "web", "sku": "MALLARD-01", "qty": 2, "currency": "GBP"}') AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"channel": null, "sku": "TEAL-02", "qty": null, "currency": null}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"sku": "EIDER-03"}') +), + +__expected__int_edge_payload_objects AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"channel":"web","qty":2,"sku":"MALLARD-01"}')::OBJECT AS order_summary, + PARSE_JSON('{"channel":"web","qty":2,"sku":"MALLARD-01"}')::OBJECT AS order_summary_keep_null, + PARSE_JSON('{"currency":"GBP"}')::OBJECT AS currency_object + UNION ALL + SELECT + 'PM-2' AS message_id, + PARSE_JSON('{"qty":null,"sku":"TEAL-02"}')::OBJECT AS order_summary, + PARSE_JSON('{"channel":null,"qty":null,"sku":"TEAL-02"}')::OBJECT AS order_summary_keep_null, + PARSE_JSON('{}')::OBJECT AS currency_object + UNION ALL + SELECT + 'PM-3' AS message_id, + PARSE_JSON('{"sku":"EIDER-03"}')::OBJECT AS order_summary, + PARSE_JSON('{"channel":null,"qty":null,"sku":"EIDER-03"}')::OBJECT AS order_summary_keep_null, + PARSE_JSON('{}')::OBJECT AS currency_object +) + +TEST (name "int_edge_payload_objects__normalises_payloads_by_inserting_deleting_and_picking_keys"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"meta": {"order_id": "DS-O-1001"}, "a.b": 1, "a": {"b": 2}, "lines": [1], ' + || '"gift_note": "Happy quacking", "pick_keys": ["sku", "lines", "missing"], ' + || '"sku": "MALLARD-01"}' + ) AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"orderId": "DS-O-0999", "meta": {}, "gift_note": null, "pick_keys": [], "sku": "TEAL-02"}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"orderId": "DS-O-1002", "Meta": {"order_id": "upper"}, "lines": null}') +), + +__expected__int_edge_payload_objects AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON( + '{"a":{"b":2},"gift_note":"Happy quacking","lines":[1],"orderId":"DS-O-1001",' + || '"pick_keys":["sku","lines","missing"],"sku":"MALLARD-01"}' + )::OBJECT AS normalised_payload, + PARSE_JSON( + '{"a":{"b":2},"gift_note":"Happy quacking","lines":[1],"orderId":"DS-O-1001",' + || '"pick_keys":["sku","lines","missing"],"sku":"MALLARD-01"}' + )::OBJECT AS gift_note_updated, + PARSE_JSON('{"lines":[1],"orderId":"DS-O-1001"}')::OBJECT AS picked_payload, + PARSE_JSON('{"lines":[1],"sku":"MALLARD-01"}')::OBJECT AS picked_by_array, + PARSE_JSON('{}')::OBJECT AS picked_by_variant, + PARSE_JSON('["a","a.b","gift_note","lines","meta","pick_keys","sku"]')::ARRAY AS payload_keys + UNION ALL + SELECT + 'PM-2' AS message_id, + PARSE_JSON('{"gift_note":null,"pick_keys":[],"sku":"TEAL-02"}')::OBJECT AS normalised_payload, + PARSE_JSON('{"pick_keys":[],"sku":"TEAL-02"}')::OBJECT AS gift_note_updated, + PARSE_JSON('{}')::OBJECT AS picked_payload, + PARSE_JSON('{}')::OBJECT AS picked_by_array, + PARSE_JSON('{}')::OBJECT AS picked_by_variant, + PARSE_JSON('["gift_note","meta","orderId","pick_keys","sku"]')::ARRAY AS payload_keys + UNION ALL + SELECT + 'PM-3' AS message_id, + PARSE_JSON('{"Meta":{"order_id":"upper"},"lines":null}')::OBJECT AS normalised_payload, + PARSE_JSON('{"Meta":{"order_id":"upper"},"lines":null}')::OBJECT AS gift_note_updated, + PARSE_JSON('{"lines":null}')::OBJECT AS picked_payload, + PARSE_JSON('{}')::OBJECT AS picked_by_array, + PARSE_JSON('{}')::OBJECT AS picked_by_variant, + PARSE_JSON('["Meta","lines","orderId"]')::ARRAY AS payload_keys +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_tags.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_tags.sql new file mode 100644 index 0000000000..f9df37cae0 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_tags.sql @@ -0,0 +1,56 @@ +TEST (name "int_edge_payload_tags__splits_tag_text_keeping_empty_tags"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + PARSE_JSON('{"tags_text": "bath,,gift"}') AS payload + UNION ALL + SELECT + 'PM-2', + PARSE_JSON('{"tags_text": ""}') + UNION ALL + SELECT + 'PM-3', + PARSE_JSON('{"tags_text": null}') + UNION ALL + SELECT + 'PM-4', + PARSE_JSON('{"tags_text": " bath , gift,"}') +), + +__expected__int_edge_payload_tags AS ( + SELECT + 'PM-1' AS message_id, + 1 AS tag_index, + 'bath' AS tag + UNION ALL + SELECT + 'PM-1' AS message_id, + 2 AS tag_index, + '' AS tag + UNION ALL + SELECT + 'PM-1' AS message_id, + 3 AS tag_index, + 'gift' AS tag + UNION ALL + SELECT + 'PM-2' AS message_id, + 1 AS tag_index, + '' AS tag + UNION ALL + SELECT + 'PM-4' AS message_id, + 1 AS tag_index, + ' bath ' AS tag + UNION ALL + SELECT + 'PM-4' AS message_id, + 2 AS tag_index, + ' gift' AS tag + UNION ALL + SELECT + 'PM-4' AS message_id, + 3 AS tag_index, + '' AS tag +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_text_payload_skus.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_text_payload_skus.sql new file mode 100644 index 0000000000..3d73d30743 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_text_payload_skus.sql @@ -0,0 +1,53 @@ +TEST (name "int_edge_text_payload_skus__groups_and_ranks_by_fields_parsed_from_text"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + '2026-03-01 09:00:00'::TIMESTAMP_NTZ AS received_at, + '{"sku": "MALLARD-01", "cart_id": "CART-1"}' AS payload_text + UNION ALL + SELECT + 'PM-2', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ, + '{"sku": "MALLARD-01", "cart_id": "CART-2"}' + UNION ALL + SELECT + 'PM-3', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ, + '{"sku": "MALLARD-01", "cart_id": "CART-2"}' + UNION ALL + SELECT + 'PM-4', + '2026-03-01 11:00:00'::TIMESTAMP_NTZ, + '{"sku": "TEAL-02", "sku": "EIDER-03"}' + UNION ALL + SELECT + 'PM-5', + '2026-03-01 12:00:00'::TIMESTAMP_NTZ, + 'not json' + UNION ALL + SELECT + 'PM-6', + '2026-03-01 08:00:00'::TIMESTAMP_NTZ, + '{"sku": "TEAL-02"}' +), + +__expected__int_edge_text_payload_skus AS ( + SELECT + 'MALLARD-01' AS sku, + 3 AS messages, + 2 AS carts, + 'PM-3' AS latest_message_id + UNION ALL + SELECT + 'TEAL-02' AS sku, + 1 AS messages, + 0 AS carts, + 'PM-6' AS latest_message_id + UNION ALL + SELECT + CAST(NULL AS VARCHAR) AS sku, + 2 AS messages, + 0 AS carts, + 'PM-5' AS latest_message_id +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_payload_events.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_payload_events.sql new file mode 100644 index 0000000000..5183f011f0 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_payload_events.sql @@ -0,0 +1,41 @@ +TEST (name "fct_edge_payload_events__publishes_event_documents_from_typed_values"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 'MKT_EU' AS partner_code, + PARSE_JSON('{"channel": "web"}') AS payload, + 19.99::FLOAT AS amount_float, + 19.99::NUMBER(18, 2) AS amount, + '2026-03-01'::DATE AS placed_on + UNION ALL + SELECT + 'PM-2', + 'MKT_EU', + PARSE_JSON('{"channel": null}'), + -0.0::FLOAT, + 0.00::NUMBER(18, 2), + NULL + UNION ALL + SELECT + 'PM-3', + 'GIFTBOX', + PARSE_JSON('{}'), + 0.1::FLOAT + 0.2::FLOAT, + 1234.50::NUMBER(18, 2), + '2028-02-29'::DATE - 1 +), + +__expected__fct_edge_payload_events AS ( + SELECT + 'PM-1' AS message_id, + '{"amount":19.99,"channel":"web","placed_on":"2026-03-01","unit_price":1.999000000000000e+01}' AS event_document_text + UNION ALL + SELECT + 'PM-2' AS message_id, + '{"amount":0,"channel":null,"placed_on":null,"unit_price":-0.000000000000000e+00}' AS event_document_text + UNION ALL + SELECT + 'PM-3' AS message_id, + '{"amount":1234.5,"channel":null,"placed_on":"2028-02-28","unit_price":3.000000000000000e-01}' AS event_document_text +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_payload_fields.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_payload_fields.sql new file mode 100644 index 0000000000..a0a991acae --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_payload_fields.sql @@ -0,0 +1,44 @@ +TEST (name "rpt_edge_partner_payload_fields__counts_json_nulls_as_present_on_the_path"); + +WITH __ref__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 'MKT_EU' AS partner_code, + PARSE_JSON('{"colour": null}') AS payload + UNION ALL + SELECT + 'PM-2', + 'MKT_EU', + PARSE_JSON('{"colour": "teal"}') + UNION ALL + SELECT + 'PM-3', + 'MKT_EU', + PARSE_JSON('{}') + UNION ALL + SELECT + 'PM-4', + 'MKT_EU', + CAST(NULL AS VARIANT) + UNION ALL + SELECT + 'PM-5', + NULL, + PARSE_JSON('{"colour": null}') +), + +__expected__rpt_edge_partner_payload_fields AS ( + SELECT + 'MKT_EU' AS partner_code, + 4 AS messages, + 1 AS colour_values, + 1 AS colour_json_nulls, + 1 AS colour_texts + UNION ALL + SELECT + 'UNKNOWN' AS partner_code, + 1 AS messages, + 0 AS colour_values, + 1 AS colour_json_nulls, + 0 AS colour_texts +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_payloads.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_payloads.sql new file mode 100644 index 0000000000..4b8aad0445 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_partner_payloads.sql @@ -0,0 +1,621 @@ +TEST (name "stg_edge_partner_payloads__keeps_latest_delivery_of_each_message"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + ' mkt_eu ' AS partner_code, + NULL AS received_at, + PARSE_JSON('{"sku": "MALLARD-01"}') AS payload, + NULL AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-1', + 'mkt_eu', + NULL, + PARSE_JSON('{"sku": "TEAL-02"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-2', + NULL, + NULL, + PARSE_JSON('{"sku": "EIDER-03"}'), + NULL, + '2026-03-02 09:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D0', + 'PM-2', + 'giftbox', + NULL, + PARSE_JSON('{"sku": "PINTAIL-04"}'), + NULL, + '2026-03-01 09:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 'MKT_EU' AS partner_code, + PARSE_JSON('{"sku":"TEAL-02"}') AS payload + UNION ALL + SELECT + 'PM-2' AS message_id, + CAST(NULL AS VARCHAR) AS partner_code, + PARSE_JSON('{"sku":"EIDER-03"}') AS payload +) + +TEST (name "stg_edge_partner_payloads__parses_lenient_and_malformed_message_text"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + NULL AS partner_code, + NULL AS received_at, + NULL AS payload, + '{"sku": "MALLARD-01", "sku": "TEAL-02"}' AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-2', + NULL, + NULL, + NULL, + '{''sku'': ''TEAL-02''}', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-3', + NULL, + NULL, + NULL, + '{"weight_kg": NaN}', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D4', + 'PM-4', + NULL, + NULL, + NULL, + '{"lines": [1, 2,]}', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D5', + 'PM-5', + NULL, + NULL, + NULL, + '"{\\"orderId\\": \\"DS-O-1001\\"}"', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D7', + 'PM-7', + NULL, + NULL, + NULL, + '{"gift": undefined}', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D8', + 'PM-8', + NULL, + NULL, + NULL, + '', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D9', + 'PM-9', + NULL, + NULL, + NULL, + 'null', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D10', + 'PM-10', + NULL, + NULL, + NULL, + '{"Sku": "EIDER-03", "sku": "PINTAIL-04"}', + '2026-03-01 10:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + CAST(NULL AS VARIANT) AS text_payload + UNION ALL + SELECT + 'PM-10' AS message_id, + PARSE_JSON('{"Sku":"EIDER-03","sku":"PINTAIL-04"}') AS text_payload + UNION ALL + SELECT + 'PM-2' AS message_id, + PARSE_JSON('{"sku":"TEAL-02"}') AS text_payload + UNION ALL + SELECT + 'PM-3' AS message_id, + PARSE_JSON('{"weight_kg":NaN}') AS text_payload + UNION ALL + SELECT + 'PM-4' AS message_id, + PARSE_JSON('{"lines":[1,2,undefined]}') AS text_payload + UNION ALL + SELECT + 'PM-5' AS message_id, + PARSE_JSON('"{\\"orderId\\": \\"DS-O-1001\\"}"') AS text_payload + UNION ALL + SELECT + 'PM-7' AS message_id, + PARSE_JSON('{"gift":null}') AS text_payload + UNION ALL + SELECT + 'PM-8' AS message_id, + CAST(NULL AS VARIANT) AS text_payload + UNION ALL + SELECT + 'PM-9' AS message_id, + PARSE_JSON('null') AS text_payload +) + +TEST (name "stg_edge_partner_payloads__reads_quantities_by_json_number_kind"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + NULL AS partner_code, + NULL AS received_at, + PARSE_JSON('{"qty": 2}') AS payload, + NULL AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-2', + NULL, + NULL, + PARSE_JSON('{"qty": 2.0}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-3', + NULL, + NULL, + PARSE_JSON('{"qty": "2"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D4', + 'PM-4', + NULL, + NULL, + PARSE_JSON('{"qty": 2e0}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D5', + 'PM-5', + NULL, + NULL, + PARSE_JSON('{"qty": 2.5}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D6', + 'PM-6', + NULL, + NULL, + PARSE_JSON('{"qty": "2.5"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D7', + 'PM-7', + NULL, + NULL, + PARSE_JSON('{"qty": -2.5}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D8', + 'PM-8', + NULL, + NULL, + PARSE_JSON('{"qty": true}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D9', + 'PM-9', + NULL, + NULL, + PARSE_JSON('{"qty": null}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D10', + 'PM-10', + NULL, + NULL, + PARSE_JSON('{}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 'INTEGER' AS qty_kind, + TRUE AS qty_is_integer, + TRUE AS qty_is_decimal, + TRUE AS qty_is_double, + 2 AS qty_units, + 2.00::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-10' AS message_id, + CAST(NULL AS VARCHAR) AS qty_kind, + CAST(NULL AS BOOLEAN) AS qty_is_integer, + CAST(NULL AS BOOLEAN) AS qty_is_decimal, + CAST(NULL AS BOOLEAN) AS qty_is_double, + CAST(NULL AS NUMBER(18, 0)) AS qty_units, + CAST(NULL AS NUMBER(18, 2)) AS qty_text_units + UNION ALL + SELECT + 'PM-2' AS message_id, + 'INTEGER' AS qty_kind, + TRUE AS qty_is_integer, + TRUE AS qty_is_decimal, + TRUE AS qty_is_double, + 2 AS qty_units, + 2.00::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-3' AS message_id, + 'VARCHAR' AS qty_kind, + FALSE AS qty_is_integer, + FALSE AS qty_is_decimal, + FALSE AS qty_is_double, + 2 AS qty_units, + 2.00::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-4' AS message_id, + 'DOUBLE' AS qty_kind, + FALSE AS qty_is_integer, + FALSE AS qty_is_decimal, + TRUE AS qty_is_double, + 2 AS qty_units, + 2.00::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-5' AS message_id, + 'DECIMAL' AS qty_kind, + FALSE AS qty_is_integer, + TRUE AS qty_is_decimal, + TRUE AS qty_is_double, + 3 AS qty_units, + 2.50::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-6' AS message_id, + 'VARCHAR' AS qty_kind, + FALSE AS qty_is_integer, + FALSE AS qty_is_decimal, + FALSE AS qty_is_double, + 3 AS qty_units, + 2.50::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-7' AS message_id, + 'DECIMAL' AS qty_kind, + FALSE AS qty_is_integer, + TRUE AS qty_is_decimal, + TRUE AS qty_is_double, + -3 AS qty_units, + -2.50::NUMBER(18, 2) AS qty_text_units + UNION ALL + SELECT + 'PM-8' AS message_id, + 'BOOLEAN' AS qty_kind, + FALSE AS qty_is_integer, + FALSE AS qty_is_decimal, + FALSE AS qty_is_double, + 1 AS qty_units, + CAST(NULL AS NUMBER(18, 2)) AS qty_text_units + UNION ALL + SELECT + 'PM-9' AS message_id, + 'NULL_VALUE' AS qty_kind, + FALSE AS qty_is_integer, + FALSE AS qty_is_decimal, + FALSE AS qty_is_double, + CAST(NULL AS NUMBER(18, 0)) AS qty_units, + CAST(NULL AS NUMBER(18, 2)) AS qty_text_units +) + +TEST (name "stg_edge_partner_payloads__casts_amounts_flags_and_dates_out_of_json"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + NULL AS partner_code, + NULL AS received_at, + PARSE_JSON('{"amount": 1.50, "gift": "yes", "placed_on": "2026-03-01"}') AS payload, + NULL AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-2', + NULL, + NULL, + PARSE_JSON('{"amount": 1E+3, "gift": true, "placed_on": "2026-02-28T23:30:00"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-3', + NULL, + NULL, + PARSE_JSON('{"amount": 2.345, "gift": "on", "placed_on": "2026-03-01"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D4', + 'PM-4', + NULL, + NULL, + PARSE_JSON('{"amount": -0, "gift": "N", "placed_on": null}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D5', + 'PM-5', + NULL, + NULL, + PARSE_JSON('{"amount": "7.125", "gift": "false"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D6', + 'PM-6', + NULL, + NULL, + PARSE_JSON('{"amount": 1e-7, "gift": "0"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 1.50::NUMBER(18, 2) AS amount, + '1.5'::FLOAT AS amount_float, + '1.5' AS amount_text, + TRUE AS is_gift, + '2026-03-01'::DATE AS placed_on + UNION ALL + SELECT + 'PM-2' AS message_id, + 1000.00::NUMBER(18, 2) AS amount, + '1000.0'::FLOAT AS amount_float, + '1000' AS amount_text, + TRUE AS is_gift, + '2026-02-28'::DATE AS placed_on + UNION ALL + SELECT + 'PM-3' AS message_id, + 2.35::NUMBER(18, 2) AS amount, + '2.345'::FLOAT AS amount_float, + '2.345' AS amount_text, + TRUE AS is_gift, + '2026-03-01'::DATE AS placed_on + UNION ALL + SELECT + 'PM-4' AS message_id, + 0.00::NUMBER(18, 2) AS amount, + '0.0'::FLOAT AS amount_float, + '0' AS amount_text, + FALSE AS is_gift, + CAST(NULL AS DATE) AS placed_on + UNION ALL + SELECT + 'PM-5' AS message_id, + 7.13::NUMBER(18, 2) AS amount, + '7.125'::FLOAT AS amount_float, + '7.125' AS amount_text, + FALSE AS is_gift, + CAST(NULL AS DATE) AS placed_on + UNION ALL + SELECT + 'PM-6' AS message_id, + 0.00::NUMBER(18, 2) AS amount, + '1e-07'::FLOAT AS amount_float, + '1e-07' AS amount_text, + FALSE AS is_gift, + CAST(NULL AS DATE) AS placed_on +) + +TEST (name "stg_edge_partner_payloads__keeps_long_json_numbers_exact"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + NULL AS partner_code, + NULL AS received_at, + PARSE_JSON('{"amount": 19.990000000000002, "txn_id": 12345678901234567890}') AS payload, + NULL AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-2', + NULL, + NULL, + PARSE_JSON('{"amount": 0.1, "txn_id": 9007199254740993}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-3', + NULL, + NULL, + PARSE_JSON('{"amount": 1234567890123456.25, "txn_id": "00042"}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + 19.99::NUMBER(18, 2) AS amount, + '19.990000000000002'::FLOAT AS amount_float, + '19.990000000000002' AS amount_text, + '12345678901234567890' AS transaction_id_text, + 12345678901234567890 AS transaction_id + UNION ALL + SELECT + 'PM-2' AS message_id, + 0.10::NUMBER(18, 2) AS amount, + '0.1'::FLOAT AS amount_float, + '0.1' AS amount_text, + '9007199254740993' AS transaction_id_text, + 9007199254740993 AS transaction_id + UNION ALL + SELECT + 'PM-3' AS message_id, + 1234567890123456.25::NUMBER(18, 2) AS amount, + '1234567890123456.2'::FLOAT AS amount_float, + '1234567890123456.25' AS amount_text, + '00042' AS transaction_id_text, + 42 AS transaction_id +) + +TEST (name "stg_edge_partner_payloads__reads_first_and_last_items_of_any_length"); + +WITH __source__edge__partner_payloads AS ( + SELECT + 'D1' AS delivery_id, + 'PM-1' AS message_id, + NULL AS partner_code, + NULL AS received_at, + PARSE_JSON('{"items": []}') AS payload, + NULL AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + UNION ALL + SELECT + 'D2', + 'PM-2', + NULL, + NULL, + PARSE_JSON('{"items": [{"sku": "MALLARD-01"}]}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D3', + 'PM-3', + NULL, + NULL, + PARSE_JSON('{"items": [{"sku": "MALLARD-01"}, {"SKU": "TEAL-02"}, {"sku": "EIDER-03"}]}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D4', + 'PM-4', + NULL, + NULL, + PARSE_JSON('{"items": null}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D5', + 'PM-5', + NULL, + NULL, + PARSE_JSON('{}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ + UNION ALL + SELECT + 'D6', + 'PM-6', + NULL, + NULL, + PARSE_JSON('{"items": [null, {"sku": "PINTAIL-04"}]}'), + NULL, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ +), + +__expected__stg_edge_partner_payloads AS ( + SELECT + 'PM-1' AS message_id, + CAST(NULL AS VARCHAR) AS first_item_sku, + CAST(NULL AS VARCHAR) AS last_item_sku, + 0 AS item_count + UNION ALL + SELECT + 'PM-2' AS message_id, + 'MALLARD-01' AS first_item_sku, + 'MALLARD-01' AS last_item_sku, + 1 AS item_count + UNION ALL + SELECT + 'PM-3' AS message_id, + 'MALLARD-01' AS first_item_sku, + 'EIDER-03' AS last_item_sku, + 3 AS item_count + UNION ALL + SELECT + 'PM-4' AS message_id, + CAST(NULL AS VARCHAR) AS first_item_sku, + CAST(NULL AS VARCHAR) AS last_item_sku, + CAST(NULL AS NUMBER(38, 0)) AS item_count + UNION ALL + SELECT + 'PM-5' AS message_id, + CAST(NULL AS VARCHAR) AS first_item_sku, + CAST(NULL AS VARCHAR) AS last_item_sku, + CAST(NULL AS NUMBER(38, 0)) AS item_count + UNION ALL + SELECT + 'PM-6' AS message_id, + CAST(NULL AS VARCHAR) AS first_item_sku, + 'PINTAIL-04' AS last_item_sku, + 2 AS item_count +) From 89143a1f0f68eb5aedfe99991ac9c1e4bbd3f23d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:15:44 +0000 Subject: [PATCH 2/6] test(benchmarks): add DuckShop edge cases for dates, times and time zones Add payment gateway callbacks and price windows to the edge-case area, with models and Snowflake-captured unit tests for: free-text times read by TRY_TO_TIMESTAMP_NTZ, TRY_TO_TIMESTAMP_TZ, TRY_TO_DATE, TRY_TO_TIME and TRY_CAST in every lenient spelling, including pre-1970 text with sub-microsecond digits; epoch milliseconds sent as JSON numbers and text; wall-clock times converted across daylight-saving gaps and overlaps; zoned times at fractional and zero offsets, and one instant at several offsets in DISTINCT; nanosecond times lowered to coarser precisions and epoch parts, past 2242 at microsecond precision; DATE_FROM_PARTS and TIMESTAMP_NTZ_FROM_PARTS normalisation; month ends, leap days and DATEDIFF boundary counting; week numbering across a year boundary; and sentinel timestamps at the edges of the calendar. --- .../edge_cases/int_edge_callback_times.sql | 56 ++++ .../int_edge_settlement_calendar.sql | 66 ++++ .../int_edge_settlement_periods.sql | 39 +++ .../edge_cases/rpt_edge_daily_callbacks.sql | 47 +++ .../edge_cases/stg_edge_gateway_callbacks.sql | 58 ++++ .../edge_cases/stg_edge_price_windows.sql | 43 +++ .../project/python/loaders/_edge_cases.py | 86 +++++ .../project/python/loaders/edge_cases.py | 10 + .../duckshop/project/sources/edge_cases.yml | 62 ++++ .../test_int_edge_callback_times.sql | 260 +++++++++++++++ .../test_int_edge_settlement_calendar.sql | 224 +++++++++++++ .../test_int_edge_settlement_periods.sql | 87 ++++++ .../test_rpt_edge_daily_callbacks.sql | 47 +++ .../test_stg_edge_gateway_callbacks.sql | 295 ++++++++++++++++++ .../test_stg_edge_price_windows.sql | 54 ++++ 15 files changed, 1434 insertions(+) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_callback_times.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_calendar.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_periods.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_daily_callbacks.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_gateway_callbacks.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_price_windows.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_callback_times.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_calendar.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_periods.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_daily_callbacks.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_gateway_callbacks.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_price_windows.sql diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_callback_times.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_callback_times.sql new file mode 100644 index 0000000000..d2d5bb434e --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_callback_times.sql @@ -0,0 +1,56 @@ +MODEL ( + description """ + One row per payment gateway callback with merchant wall-clock times converted across + daylight-saving changes, zoned times read at their own offsets, and nanosecond times lowered to + coarser precisions and epoch parts + """, + tags [edge_case], + columns ( + callback_id (type VARCHAR, nullable false, audits [not_null, unique]), + wall_time_utc (type TIMESTAMP_NTZ), + wall_time_as_zoned (type TIMESTAMP_TZ), + event_at_utc (type TIMESTAMP_NTZ), + event_at_merchant (type TIMESTAMP_TZ), + event_local_date (type DATE), + event_ns_date (type DATE), + event_ns_ms (type TIMESTAMP_NTZ(3)), + event_ns_us (type TIMESTAMP_NTZ(6)), + event_ns_text (type VARCHAR), + epoch_second (type NUMBER(38, 0)), + epoch_millisecond (type NUMBER(38, 0)), + epoch_microsecond (type NUMBER(38, 0)), + epoch_nanosecond (type NUMBER(38, 0)), + ), +); + +WITH callbacks AS ( + SELECT + s.callback_id, + s.merchant_zone, + s.local_wall_time, + s.event_at, + s.event_at_ns + FROM __ref("stg_edge_gateway_callbacks") AS s +), + +final AS ( + SELECT + c.callback_id, + CONVERT_TIMEZONE(c.merchant_zone, 'UTC', c.local_wall_time)::TIMESTAMP_NTZ AS wall_time_utc, + c.local_wall_time::TIMESTAMP_TZ AS wall_time_as_zoned, + CONVERT_TIMEZONE('UTC', c.event_at)::TIMESTAMP_NTZ AS event_at_utc, + CONVERT_TIMEZONE(c.merchant_zone, c.event_at)::TIMESTAMP_TZ AS event_at_merchant, + c.event_at::DATE AS event_local_date, + c.event_at_ns::DATE AS event_ns_date, + c.event_at_ns::TIMESTAMP_NTZ(3) AS event_ns_ms, + c.event_at_ns::TIMESTAMP_NTZ(6) AS event_ns_us, + TO_VARCHAR(c.event_at_ns, 'YYYY-MM-DD HH24:MI:SS.FF3')::VARCHAR AS event_ns_text, + DATE_PART(EPOCH_SECOND, c.event_at_ns)::NUMBER(38, 0) AS epoch_second, + DATE_PART(EPOCH_MILLISECOND, c.event_at_ns)::NUMBER(38, 0) AS epoch_millisecond, + DATE_PART(EPOCH_MICROSECOND, c.event_at_ns)::NUMBER(38, 0) AS epoch_microsecond, + DATE_PART(EPOCH_NANOSECOND, c.event_at_ns)::NUMBER(38, 0) AS epoch_nanosecond + FROM callbacks AS c +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_calendar.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_calendar.sql new file mode 100644 index 0000000000..7ae6557ec1 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_calendar.sql @@ -0,0 +1,66 @@ +MODEL ( + description """ + One row per payment gateway callback with its business date placed in the settlement calendar: + month ends built from out-of-range date parts, renewal dates a month on, and the day, ISO and + week-of-year numbering around a year boundary + """, + tags [edge_case], + columns ( + callback_id (type VARCHAR, nullable false, audits [not_null, unique]), + business_date (type DATE), + month_end (type DATE), + previous_month_end (type DATE), + next_year_start (type DATE), + day_before_month (type DATE), + end_of_day (type TIMESTAMP_NTZ), + renews_on (type DATE), + renews_on_add_months (type DATE), + last_day_of_month (type DATE), + day_of_week (type INT), + iso_day_of_week (type INT), + week_of_year (type INT), + iso_week (type INT), + year_of_week (type INT), + iso_year_of_week (type INT), + week_start (type DATE), + ), +); + +WITH callbacks AS ( + SELECT + s.callback_id, + s.business_date + FROM __ref("stg_edge_gateway_callbacks") AS s +), + +final AS ( + SELECT + c.callback_id, + c.business_date, + DATE_FROM_PARTS(YEAR(c.business_date), MONTH(c.business_date) + 1, 0) AS month_end, + DATE_FROM_PARTS(YEAR(c.business_date), MONTH(c.business_date), 0) AS previous_month_end, + DATE_FROM_PARTS(YEAR(c.business_date), 13, 1) AS next_year_start, + DATE_FROM_PARTS(YEAR(c.business_date), MONTH(c.business_date), -1) AS day_before_month, + TIMESTAMP_NTZ_FROM_PARTS( + YEAR(c.business_date), + MONTH(c.business_date), + DAY(c.business_date), + 24, + 0, + 0 + ) AS end_of_day, + DATEADD('month', 1, c.business_date)::DATE AS renews_on, + ADD_MONTHS(c.business_date, 1)::DATE AS renews_on_add_months, + LAST_DAY(c.business_date)::DATE AS last_day_of_month, + DAYOFWEEK(c.business_date)::INT AS day_of_week, + DAYOFWEEKISO(c.business_date)::INT AS iso_day_of_week, + WEEKOFYEAR(c.business_date)::INT AS week_of_year, + WEEKISO(c.business_date)::INT AS iso_week, + YEAROFWEEK(c.business_date)::INT AS year_of_week, + YEAROFWEEKISO(c.business_date)::INT AS iso_year_of_week, + DATE_TRUNC('week', c.business_date)::DATE AS week_start + FROM callbacks AS c +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_periods.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_periods.sql new file mode 100644 index 0000000000..daccbdd7a5 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_settlement_periods.sql @@ -0,0 +1,39 @@ +MODEL ( + description """ + One row per payment gateway callback with its settlement period measured by calendar boundaries + crossed, not elapsed time, in days, months and minutes + """, + tags [edge_case], + columns ( + callback_id (type VARCHAR, nullable false, audits [not_null, unique]), + days_crossed (type INT), + months_crossed (type INT), + hours_crossed (type INT), + minutes_elapsed (type INT), + days_backwards (type INT), + period_end_month (type DATE), + ), +); + +WITH callbacks AS ( + SELECT + s.callback_id, + s.period_start, + s.period_end + FROM __ref("stg_edge_gateway_callbacks") AS s +), + +final AS ( + SELECT + c.callback_id, + DATEDIFF('day', c.period_start, c.period_end)::INT AS days_crossed, + DATEDIFF('month', c.period_start, c.period_end)::INT AS months_crossed, + DATEDIFF('hour', c.period_start, c.period_end)::INT AS hours_crossed, + TIMESTAMPDIFF('minute', c.period_start, c.period_end)::INT AS minutes_elapsed, + DATEDIFF('day', c.period_end, c.period_start)::INT AS days_backwards, + DATE_TRUNC('month', c.period_end)::DATE AS period_end_month + FROM callbacks AS c +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_daily_callbacks.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_daily_callbacks.sql new file mode 100644 index 0000000000..da323735af --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_daily_callbacks.sql @@ -0,0 +1,47 @@ +MODEL ( + description """ + One row per local callback date, counting callbacks and the distinct instants among them, where + the same instant can arrive at different UTC offsets + """, + tags [edge_case], + columns ( + event_local_date (type DATE, nullable false, audits [not_null, unique]), + callbacks (type INT), + distinct_instants (type INT), + distinct_offsets (type INT), + first_event_utc (type TIMESTAMP_NTZ), + ), +); + +WITH callback_times AS ( + SELECT + t.callback_id, + t.event_local_date, + t.event_at_utc + FROM __ref("int_edge_callback_times") AS t +), + +callbacks AS ( + SELECT + s.callback_id, + s.event_at + FROM __ref("stg_edge_gateway_callbacks") AS s +), + +final AS ( + SELECT + t.event_local_date, + COUNT(*)::INT AS callbacks, + COUNT(DISTINCT c.event_at)::INT AS distinct_instants, + COUNT(DISTINCT TO_VARCHAR(c.event_at, 'TZH:TZM'))::INT AS distinct_offsets, + MIN(t.event_at_utc) AS first_event_utc + FROM callback_times AS t + INNER JOIN callbacks AS c + ON t.callback_id = c.callback_id + WHERE t.event_local_date IS NOT NULL + GROUP BY + t.event_local_date +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_gateway_callbacks.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_gateway_callbacks.sql new file mode 100644 index 0000000000..abfcfc4694 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_gateway_callbacks.sql @@ -0,0 +1,58 @@ +MODEL ( + description """ + One row per payment gateway callback, with its free-text time read in every lenient form and its + epoch milliseconds read as milliseconds, as epoch text and as a plain cast + """, + tags [edge_case], + columns ( + callback_id (type VARCHAR, nullable false, audits [not_null, unique]), + merchant_zone (type VARCHAR), + local_wall_time (type TIMESTAMP_NTZ), + event_at (type TIMESTAMP_TZ), + event_at_ns (type TIMESTAMP_NTZ(9)), + event_time_text (type VARCHAR), + business_date (type DATE), + period_start (type TIMESTAMP_NTZ), + period_end (type TIMESTAMP_NTZ), + text_timestamp (type TIMESTAMP_NTZ), + text_timestamp_tz (type TIMESTAMP_TZ), + text_date (type DATE), + text_time (type TIME), + text_date_cast (type DATE), + text_timestamp_day_month (type TIMESTAMP_NTZ), + sent_at_from_ms (type TIMESTAMP_NTZ), + sent_at_from_text (type TIMESTAMP_NTZ), + sent_at_cast (type TIMESTAMP_NTZ), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__gateway_callbacks") +), + +final AS ( + SELECT + s.callback_id, + s.merchant_zone, + s.local_wall_time, + s.event_at, + s.event_at_ns, + s.event_time_text, + s.business_date, + s.period_start, + s.period_end, + TRY_TO_TIMESTAMP_NTZ(s.event_time_text) AS text_timestamp, + TRY_TO_TIMESTAMP_TZ(s.event_time_text) AS text_timestamp_tz, + TRY_TO_DATE(s.event_time_text) AS text_date, + TRY_TO_TIME(s.event_time_text)::TIME AS text_time, + TRY_CAST(s.event_time_text AS DATE) AS text_date_cast, + TRY_TO_TIMESTAMP_NTZ(s.event_time_text, 'DD-MON-YY HH24:MI') AS text_timestamp_day_month, + TO_TIMESTAMP_NTZ(s.body:ts_ms::NUMBER(38, 0), 3) AS sent_at_from_ms, + TRY_TO_TIMESTAMP_NTZ(s.body:ts_ms::VARCHAR) AS sent_at_from_text, + s.body:ts_ms::TIMESTAMP_NTZ AS sent_at_cast + FROM src AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_price_windows.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_price_windows.sql new file mode 100644 index 0000000000..51a2626f57 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_price_windows.sql @@ -0,0 +1,43 @@ +MODEL ( + description """ + One row per SKU price window, with open ends closed by sentinel timestamps at microsecond + precision so the window always has two bounds + """, + tags [edge_case], + columns ( + window_id (type VARCHAR, nullable false, audits [not_null, unique]), + sku (type VARCHAR), + price_minor (type NUMBER(38, 0)), + valid_from (type TIMESTAMP_NTZ(6)), + valid_to (type TIMESTAMP_NTZ(6)), + is_open_ended (type BOOLEAN), + window_days (type INT), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__price_windows") +), + +final AS ( + SELECT + s.window_id, + s.sku, + s.price_minor, + COALESCE(s.valid_from, '0001-01-01 00:00:00'::TIMESTAMP_NTZ(6)) AS valid_from, + COALESCE(s.valid_to, '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6)) AS valid_to, + ( + s.valid_to IS NULL + OR s.valid_to >= '9999-01-01'::TIMESTAMP_NTZ(6) + ) AS is_open_ended, + DATEDIFF( + 'day', + COALESCE(s.valid_from, '0001-01-01 00:00:00'::TIMESTAMP_NTZ(6)), + COALESCE(s.valid_to, '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6)) + )::INT AS window_days + FROM src AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/python/loaders/_edge_cases.py b/benchmarks/duckshop/project/python/loaders/_edge_cases.py index bc30f42dd7..958f475634 100644 --- a/benchmarks/duckshop/project/python/loaders/_edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/_edge_cases.py @@ -90,6 +90,92 @@ def partner_payloads() -> Rows: return rows +def gateway_callbacks() -> Rows: + callbacks = [ + # zone, wall time, zoned event, nanosecond event, free text, business date, period, body + ( + "Europe/London", + "2026-03-29 01:30:00", + "2026-03-01 10:00:00 +05:45", + "2026-03-01 23:59:59.9999995", + "2026-03-01T10:15:00Z", + "2026-01-31", + ("2026-03-31 23:59:00", "2026-04-01 00:01:00"), + '{"ts_ms": 1772359200000}', + ), + ( + "Europe/London", + "2026-10-25 01:30:00", + "2026-03-01 04:15:00 -00:00", + "2026-03-01 10:00:00.0005", + "01-MAR-26 10:15", + "2028-02-29", + ("2026-01-31 00:00:00", "2026-02-01 00:00:00"), + '{"ts_ms": "1772359200000"}', + ), + ( + "America/New_York", + "2026-11-01 01:30:00", + "2026-02-28 18:45:00 -09:30", + "1962-07-01 12:00:00.123456789", + "2026-03-01 24:00:00", + "2026-12-27", + ("2028-02-28 12:00:00", "2028-03-01 12:00:00"), + '{"ts_ms": 1772359200}', + ), + ( + "Australia/Sydney", + "2026-04-05 02:30:00", + "2026-03-01 23:30:00 +05:45", + "2250-06-01 00:00:00.123456", + " 2026-3-1 7:5:0 ", + "2027-01-03", + ("2026-03-01 10:00:59", "2026-03-01 10:01:00"), + "{}", + ), + ] + rows: Rows = [] + for index, (zone, wall, zoned, nano, text, business, period, body) in enumerate(callbacks): + rows.append( + { + "callback_id": f"CB-{index + 1:04d}", + "merchant_zone": zone, + "local_wall_time": wall, + "event_at": zoned, + "event_at_ns": nano, + "event_time_text": text, + "business_date": business, + "period_start": period[0], + "period_end": period[1], + "body": body, + "_loaded_at": _at("2026-03-02 00:00:00"), + } + ) + return rows + + +def price_windows() -> Rows: + windows = [ + ("MALLARD-01", 1299, None, None), + ("TEAL-02", 899, "0001-01-01 00:00:00", "9999-12-31 23:59:59.999999"), + ("EIDER-03", 2450, "2026-01-01 00:00:00", None), + ("PINTAIL-04", 1575, "1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000001"), + ] + return [ + { + "window_id": f"PW-{index + 1:04d}", + "sku": sku, + "price_minor": price, + "valid_from": valid_from, + "valid_to": valid_to, + "_loaded_at": _at("2026-03-02 00:00:00"), + } + for index, (sku, price, valid_from, valid_to) in enumerate(windows) + ] + + TABLES: dict[str, Rows] = { "edge__partner_payloads": partner_payloads(), + "edge__gateway_callbacks": gateway_callbacks(), + "edge__price_windows": price_windows(), } diff --git a/benchmarks/duckshop/project/python/loaders/edge_cases.py b/benchmarks/duckshop/project/python/loaders/edge_cases.py index fe5a743530..8f931ba966 100644 --- a/benchmarks/duckshop/project/python/loaders/edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/edge_cases.py @@ -21,3 +21,13 @@ def _load(ctx: LoaderContext, source_name: str) -> object: @loader def edge__partner_payloads(ctx: LoaderContext) -> object: return _load(ctx, "edge__partner_payloads") + + +@loader +def edge__gateway_callbacks(ctx: LoaderContext) -> object: + return _load(ctx, "edge__gateway_callbacks") + + +@loader +def edge__price_windows(ctx: LoaderContext) -> object: + return _load(ctx, "edge__price_windows") diff --git a/benchmarks/duckshop/project/sources/edge_cases.yml b/benchmarks/duckshop/project/sources/edge_cases.yml index 6f68757c46..e30a0f265b 100644 --- a/benchmarks/duckshop/project/sources/edge_cases.yml +++ b/benchmarks/duckshop/project/sources/edge_cases.yml @@ -32,3 +32,65 @@ sources: nullable: false audits: - not_null + - name: edge__gateway_callbacks + description: >- + Append-only payment gateway callbacks with timestamps as merchant wall-clock times, zoned + values, nanosecond values, free text and epoch numbers in JSON + managed: true + contract: enforced + columns: + - name: callback_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: merchant_zone + type: VARCHAR + - name: local_wall_time + type: TIMESTAMP_NTZ + - name: event_at + type: TIMESTAMP_TZ + - name: event_at_ns + type: TIMESTAMP_NTZ(9) + - name: event_time_text + type: VARCHAR + - name: business_date + type: DATE + - name: period_start + type: TIMESTAMP_NTZ + - name: period_end + type: TIMESTAMP_NTZ + - name: body + type: VARIANT + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null + - name: edge__price_windows + description: >- + Price list windows per SKU whose open ends are written as sentinel dates at the edges of the + calendar + managed: true + contract: enforced + columns: + - name: window_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: sku + type: VARCHAR + - name: price_minor + type: NUMBER(38,0) + - name: valid_from + type: TIMESTAMP_NTZ(6) + - name: valid_to + type: TIMESTAMP_NTZ(6) + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_callback_times.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_callback_times.sql new file mode 100644 index 0000000000..722c46d92c --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_callback_times.sql @@ -0,0 +1,260 @@ +TEST (name "int_edge_callback_times__converts_wall_times_across_daylight_saving_changes"); + +WITH __ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + v.merchant_zone, + v.local_wall_time::TIMESTAMP_NTZ AS local_wall_time + FROM VALUES + ('CB-01', 'Europe/London', '2026-03-29 00:30:00'), + ('CB-02', 'Europe/London', '2026-03-29 01:30:00'), + ('CB-03', 'Europe/London', '2026-03-29 02:30:00'), + ('CB-04', 'Europe/London', '2026-10-25 01:30:00'), + ('CB-05', 'America/New_York', '2026-03-08 02:30:00'), + ('CB-06', 'America/New_York', '2026-11-01 01:30:00'), + ('CB-07', 'Australia/Sydney', '2026-04-05 02:30:00'), + ('CB-08', 'Australia/Sydney', '2026-10-04 02:30:00'), + ('CB-09', 'Asia/Kathmandu', '2026-03-01 10:00:00'), + ('CB-10', 'Pacific/Marquesas', '2026-03-01 10:00:00') AS v(callback_id, merchant_zone, local_wall_time) +), + +__expected__int_edge_callback_times AS ( + SELECT + 'CB-01' AS callback_id, + '2026-03-29 00:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-29 00:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2026-03-29 01:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-29 01:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-03' AS callback_id, + '2026-03-29 01:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-29 02:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-04' AS callback_id, + '2026-10-25 00:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-10-25 01:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-05' AS callback_id, + '2026-03-08 07:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-08 02:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-06' AS callback_id, + '2026-11-01 05:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-11-01 01:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-07' AS callback_id, + '2026-04-04 15:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-04-05 02:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-08' AS callback_id, + '2026-10-03 16:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-10-04 02:30:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-09' AS callback_id, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-01 10:00:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned + UNION ALL + SELECT + 'CB-10' AS callback_id, + '2026-03-01 19:30:00'::TIMESTAMP_NTZ(9) AS wall_time_utc, + '2026-03-01 10:00:00 Z'::TIMESTAMP_TZ(9) AS wall_time_as_zoned +) + +TEST (name "int_edge_callback_times__reads_zoned_times_at_fractional_and_zero_offsets"); + +WITH __ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + v.merchant_zone, + NULL AS local_wall_time, + v.event_at::TIMESTAMP_TZ AS event_at + FROM VALUES + ('CB-01', 'Asia/Kathmandu', '2026-03-01 10:00:00 +05:45'), + ('CB-02', 'Europe/London', '2026-03-01 04:15:00 +00:00'), + ('CB-03', 'Europe/London', '2026-03-01 04:15:00 -00:00'), + ('CB-04', 'Pacific/Marquesas', '2026-02-28 18:45:00 -09:30'), + ('CB-05', 'Asia/Kathmandu', '2026-03-01 23:30:00 +05:45'), + ('CB-06', 'America/New_York', '2026-03-01 00:30:00 +14:00') AS v(callback_id, merchant_zone, event_at) +), + +__expected__int_edge_callback_times AS ( + SELECT + 'CB-01' AS callback_id, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-03-01 10:00:00 +05:45'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-03-01'::DATE AS event_local_date + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-03-01 04:15:00 Z'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-03-01'::DATE AS event_local_date + UNION ALL + SELECT + 'CB-03' AS callback_id, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-03-01 04:15:00 Z'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-03-01'::DATE AS event_local_date + UNION ALL + SELECT + 'CB-04' AS callback_id, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-02-28 18:45:00 -09:30'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-02-28'::DATE AS event_local_date + UNION ALL + SELECT + 'CB-05' AS callback_id, + '2026-03-01 17:45:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-03-01 23:30:00 +05:45'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-03-01'::DATE AS event_local_date + UNION ALL + SELECT + 'CB-06' AS callback_id, + '2026-02-28 10:30:00'::TIMESTAMP_NTZ(9) AS event_at_utc, + '2026-02-28 05:30:00 -05:00'::TIMESTAMP_TZ(9) AS event_at_merchant, + '2026-03-01'::DATE AS event_local_date +) + +TEST (name "int_edge_callback_times__lowers_nanosecond_times_by_truncating"); + +WITH __ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + NULL AS event_at, + v.event_at_ns::TIMESTAMP_NTZ(9) AS event_at_ns + FROM VALUES + ('CB-01', '2026-03-01 23:59:59.9999995'), + ('CB-02', '2026-03-01 10:00:00.0005'), + ('CB-03', '2026-03-01 10:00:00.0015'), + ('CB-04', '2026-03-01 10:00:00.000000001'), + ('CB-05', '1962-07-01 12:00:00.123456789'), + ('CB-06', '1969-12-31 23:59:59.9999999'), + ('CB-07', '2250-06-01 00:00:00.123456'), + ('CB-08', '2300-01-01 00:00:00.5'), + ('CB-09', '0001-01-01 00:00:00'), + ('CB-10', '9999-12-31 23:59:59.999999') AS v(callback_id, event_at_ns) +), + +__expected__int_edge_callback_times AS ( + SELECT + 'CB-01' AS callback_id, + '2026-03-01'::DATE AS event_ns_date, + '2026-03-01 23:59:59.999'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2026-03-01 23:59:59.999999'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2026-03-01 23:59:59.999' AS event_ns_text, + 1772409599 AS epoch_second, + 1772409599999 AS epoch_millisecond, + 1772409599999999 AS epoch_microsecond, + 1772409599999999500 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2026-03-01'::DATE AS event_ns_date, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2026-03-01 10:00:00.0005'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2026-03-01 10:00:00.000' AS event_ns_text, + 1772359200 AS epoch_second, + 1772359200000 AS epoch_millisecond, + 1772359200000500 AS epoch_microsecond, + 1772359200000500000 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-03' AS callback_id, + '2026-03-01'::DATE AS event_ns_date, + '2026-03-01 10:00:00.001'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2026-03-01 10:00:00.0015'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2026-03-01 10:00:00.001' AS event_ns_text, + 1772359200 AS epoch_second, + 1772359200001 AS epoch_millisecond, + 1772359200001500 AS epoch_microsecond, + 1772359200001500000 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-04' AS callback_id, + '2026-03-01'::DATE AS event_ns_date, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2026-03-01 10:00:00.000' AS event_ns_text, + 1772359200 AS epoch_second, + 1772359200000 AS epoch_millisecond, + 1772359200000000 AS epoch_microsecond, + 1772359200000000001 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-05' AS callback_id, + '1962-07-01'::DATE AS event_ns_date, + '1962-07-01 12:00:00.123'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '1962-07-01 12:00:00.123456'::TIMESTAMP_NTZ(6) AS event_ns_us, + '1962-07-01 12:00:00.123' AS event_ns_text, + -236779200 AS epoch_second, + -236779199877 AS epoch_millisecond, + -236779199876544 AS epoch_microsecond, + -236779199876543211 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-06' AS callback_id, + '1969-12-31'::DATE AS event_ns_date, + '1969-12-31 23:59:59.999'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '1969-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS event_ns_us, + '1969-12-31 23:59:59.999' AS event_ns_text, + -1 AS epoch_second, + -1 AS epoch_millisecond, + -1 AS epoch_microsecond, + -100 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-07' AS callback_id, + '2250-06-01'::DATE AS event_ns_date, + '2250-06-01 00:00:00.123'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2250-06-01 00:00:00.123456'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2250-06-01 00:00:00.123' AS event_ns_text, + 8849001600 AS epoch_second, + 8849001600123 AS epoch_millisecond, + 8849001600123456 AS epoch_microsecond, + 8849001600123456000 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-08' AS callback_id, + '2300-01-01'::DATE AS event_ns_date, + '2300-01-01 00:00:00.5'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '2300-01-01 00:00:00.5'::TIMESTAMP_NTZ(6) AS event_ns_us, + '2300-01-01 00:00:00.500' AS event_ns_text, + 10413792000 AS epoch_second, + 10413792000500 AS epoch_millisecond, + 10413792000500000 AS epoch_microsecond, + 10413792000500000000 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-09' AS callback_id, + '0001-01-01'::DATE AS event_ns_date, + '0001-01-01 00:00:00'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '0001-01-01 00:00:00'::TIMESTAMP_NTZ(6) AS event_ns_us, + '0001-01-01 00:00:00.000' AS event_ns_text, + -62135596800 AS epoch_second, + -62135596800000 AS epoch_millisecond, + -62135596800000000 AS epoch_microsecond, + -62135596800000000000 AS epoch_nanosecond + UNION ALL + SELECT + 'CB-10' AS callback_id, + '9999-12-31'::DATE AS event_ns_date, + '9999-12-31 23:59:59.999'::TIMESTAMP_NTZ(3) AS event_ns_ms, + '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS event_ns_us, + '9999-12-31 23:59:59.999' AS event_ns_text, + 253402300799 AS epoch_second, + 253402300799999 AS epoch_millisecond, + 253402300799999999 AS epoch_microsecond, + 253402300799999999000 AS epoch_nanosecond +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_calendar.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_calendar.sql new file mode 100644 index 0000000000..df57636741 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_calendar.sql @@ -0,0 +1,224 @@ +TEST (name "int_edge_settlement_calendar__normalises_date_parts_and_numbers_weeks_across_years"); + +WITH __ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + NULL AS event_at, + NULL AS event_at_ns, + NULL AS event_time_text, + v.business_date::DATE AS business_date + FROM VALUES + ('CB-01', '2026-01-31'), + ('CB-02', '2028-01-31'), + ('CB-03', '2028-02-29'), + ('CB-04', '2026-03-31'), + ('CB-05', '2026-12-27'), + ('CB-06', '2026-12-31'), + ('CB-07', '2027-01-01'), + ('CB-08', '2027-01-03'), + ('CB-09', '2027-01-04'), + ('CB-10', '2024-12-30'), + ('CB-11', '2026-12-01') AS v(callback_id, business_date) +), + +__expected__int_edge_settlement_calendar AS ( + SELECT + 'CB-01' AS callback_id, + '2026-01-31'::DATE AS month_end, + '2025-12-31'::DATE AS previous_month_end, + '2027-01-01'::DATE AS next_year_start, + '2025-12-30'::DATE AS day_before_month, + '2026-02-01 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2026-02-28'::DATE AS renews_on, + '2026-02-28'::DATE AS renews_on_add_months, + '2026-01-31'::DATE AS last_day_of_month, + 6 AS day_of_week, + 6 AS iso_day_of_week, + 5 AS week_of_year, + 5 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-01-26'::DATE AS week_start + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2028-01-31'::DATE AS month_end, + '2027-12-31'::DATE AS previous_month_end, + '2029-01-01'::DATE AS next_year_start, + '2027-12-30'::DATE AS day_before_month, + '2028-02-01 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2028-02-29'::DATE AS renews_on, + '2028-02-29'::DATE AS renews_on_add_months, + '2028-01-31'::DATE AS last_day_of_month, + 1 AS day_of_week, + 1 AS iso_day_of_week, + 5 AS week_of_year, + 5 AS iso_week, + 2028 AS year_of_week, + 2028 AS iso_year_of_week, + '2028-01-31'::DATE AS week_start + UNION ALL + SELECT + 'CB-03' AS callback_id, + '2028-02-29'::DATE AS month_end, + '2028-01-31'::DATE AS previous_month_end, + '2029-01-01'::DATE AS next_year_start, + '2028-01-30'::DATE AS day_before_month, + '2028-03-01 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2028-03-29'::DATE AS renews_on, + '2028-03-31'::DATE AS renews_on_add_months, + '2028-02-29'::DATE AS last_day_of_month, + 2 AS day_of_week, + 2 AS iso_day_of_week, + 9 AS week_of_year, + 9 AS iso_week, + 2028 AS year_of_week, + 2028 AS iso_year_of_week, + '2028-02-28'::DATE AS week_start + UNION ALL + SELECT + 'CB-04' AS callback_id, + '2026-03-31'::DATE AS month_end, + '2026-02-28'::DATE AS previous_month_end, + '2027-01-01'::DATE AS next_year_start, + '2026-02-27'::DATE AS day_before_month, + '2026-04-01 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2026-04-30'::DATE AS renews_on, + '2026-04-30'::DATE AS renews_on_add_months, + '2026-03-31'::DATE AS last_day_of_month, + 2 AS day_of_week, + 2 AS iso_day_of_week, + 14 AS week_of_year, + 14 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-03-30'::DATE AS week_start + UNION ALL + SELECT + 'CB-05' AS callback_id, + '2026-12-31'::DATE AS month_end, + '2026-11-30'::DATE AS previous_month_end, + '2027-01-01'::DATE AS next_year_start, + '2026-11-29'::DATE AS day_before_month, + '2026-12-28 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-01-27'::DATE AS renews_on, + '2027-01-27'::DATE AS renews_on_add_months, + '2026-12-31'::DATE AS last_day_of_month, + 0 AS day_of_week, + 7 AS iso_day_of_week, + 52 AS week_of_year, + 52 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-12-21'::DATE AS week_start + UNION ALL + SELECT + 'CB-06' AS callback_id, + '2026-12-31'::DATE AS month_end, + '2026-11-30'::DATE AS previous_month_end, + '2027-01-01'::DATE AS next_year_start, + '2026-11-29'::DATE AS day_before_month, + '2027-01-01 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-01-31'::DATE AS renews_on, + '2027-01-31'::DATE AS renews_on_add_months, + '2026-12-31'::DATE AS last_day_of_month, + 4 AS day_of_week, + 4 AS iso_day_of_week, + 53 AS week_of_year, + 53 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-12-28'::DATE AS week_start + UNION ALL + SELECT + 'CB-07' AS callback_id, + '2027-01-31'::DATE AS month_end, + '2026-12-31'::DATE AS previous_month_end, + '2028-01-01'::DATE AS next_year_start, + '2026-12-30'::DATE AS day_before_month, + '2027-01-02 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-02-01'::DATE AS renews_on, + '2027-02-01'::DATE AS renews_on_add_months, + '2027-01-31'::DATE AS last_day_of_month, + 5 AS day_of_week, + 5 AS iso_day_of_week, + 53 AS week_of_year, + 53 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-12-28'::DATE AS week_start + UNION ALL + SELECT + 'CB-08' AS callback_id, + '2027-01-31'::DATE AS month_end, + '2026-12-31'::DATE AS previous_month_end, + '2028-01-01'::DATE AS next_year_start, + '2026-12-30'::DATE AS day_before_month, + '2027-01-04 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-02-03'::DATE AS renews_on, + '2027-02-03'::DATE AS renews_on_add_months, + '2027-01-31'::DATE AS last_day_of_month, + 0 AS day_of_week, + 7 AS iso_day_of_week, + 53 AS week_of_year, + 53 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-12-28'::DATE AS week_start + UNION ALL + SELECT + 'CB-09' AS callback_id, + '2027-01-31'::DATE AS month_end, + '2026-12-31'::DATE AS previous_month_end, + '2028-01-01'::DATE AS next_year_start, + '2026-12-30'::DATE AS day_before_month, + '2027-01-05 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-02-04'::DATE AS renews_on, + '2027-02-04'::DATE AS renews_on_add_months, + '2027-01-31'::DATE AS last_day_of_month, + 1 AS day_of_week, + 1 AS iso_day_of_week, + 1 AS week_of_year, + 1 AS iso_week, + 2027 AS year_of_week, + 2027 AS iso_year_of_week, + '2027-01-04'::DATE AS week_start + UNION ALL + SELECT + 'CB-10' AS callback_id, + '2024-12-31'::DATE AS month_end, + '2024-11-30'::DATE AS previous_month_end, + '2025-01-01'::DATE AS next_year_start, + '2024-11-29'::DATE AS day_before_month, + '2024-12-31 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2025-01-30'::DATE AS renews_on, + '2025-01-30'::DATE AS renews_on_add_months, + '2024-12-31'::DATE AS last_day_of_month, + 1 AS day_of_week, + 1 AS iso_day_of_week, + 1 AS week_of_year, + 1 AS iso_week, + 2025 AS year_of_week, + 2025 AS iso_year_of_week, + '2024-12-30'::DATE AS week_start + UNION ALL + SELECT + 'CB-11' AS callback_id, + '2026-12-31'::DATE AS month_end, + '2026-11-30'::DATE AS previous_month_end, + '2027-01-01'::DATE AS next_year_start, + '2026-11-29'::DATE AS day_before_month, + '2026-12-02 00:00:00'::TIMESTAMP_NTZ(9) AS end_of_day, + '2027-01-01'::DATE AS renews_on, + '2027-01-01'::DATE AS renews_on_add_months, + '2026-12-31'::DATE AS last_day_of_month, + 2 AS day_of_week, + 2 AS iso_day_of_week, + 49 AS week_of_year, + 49 AS iso_week, + 2026 AS year_of_week, + 2026 AS iso_year_of_week, + '2026-11-30'::DATE AS week_start +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_periods.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_periods.sql new file mode 100644 index 0000000000..598e52f916 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_settlement_periods.sql @@ -0,0 +1,87 @@ +TEST (name "int_edge_settlement_periods__counts_calendar_boundaries_crossed"); + +WITH __ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + NULL AS event_at, + NULL AS event_at_ns, + NULL AS event_time_text, + NULL AS business_date, + v.period_start::TIMESTAMP_NTZ AS period_start, + v.period_end::TIMESTAMP_NTZ AS period_end + FROM VALUES + ('CB-01', '2026-03-31 23:59:00', '2026-04-01 00:01:00'), + ('CB-02', '2026-01-31 00:00:00', '2026-02-01 00:00:00'), + ('CB-03', '2026-01-01 00:00:00', '2026-12-31 23:59:59'), + ('CB-04', '2028-02-28 12:00:00', '2028-03-01 12:00:00'), + ('CB-05', '2026-03-01 10:00:59', '2026-03-01 10:01:00'), + ('CB-06', '2026-03-29 00:30:00', '2026-03-29 03:30:00'), + ('CB-07', '2026-12-31 23:00:00', '2027-01-01 01:00:00') AS v(callback_id, period_start, period_end) +), + +__expected__int_edge_settlement_periods AS ( + SELECT + 'CB-01' AS callback_id, + 1 AS days_crossed, + 1 AS months_crossed, + 1 AS hours_crossed, + 2 AS minutes_elapsed, + -1 AS days_backwards, + '2026-04-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-02' AS callback_id, + 1 AS days_crossed, + 1 AS months_crossed, + 24 AS hours_crossed, + 1440 AS minutes_elapsed, + -1 AS days_backwards, + '2026-02-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-03' AS callback_id, + 364 AS days_crossed, + 11 AS months_crossed, + 8759 AS hours_crossed, + 525599 AS minutes_elapsed, + -364 AS days_backwards, + '2026-12-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-04' AS callback_id, + 2 AS days_crossed, + 1 AS months_crossed, + 48 AS hours_crossed, + 2880 AS minutes_elapsed, + -2 AS days_backwards, + '2028-03-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-05' AS callback_id, + 0 AS days_crossed, + 0 AS months_crossed, + 0 AS hours_crossed, + 1 AS minutes_elapsed, + 0 AS days_backwards, + '2026-03-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-06' AS callback_id, + 0 AS days_crossed, + 0 AS months_crossed, + 3 AS hours_crossed, + 180 AS minutes_elapsed, + 0 AS days_backwards, + '2026-03-01'::DATE AS period_end_month + UNION ALL + SELECT + 'CB-07' AS callback_id, + 1 AS days_crossed, + 1 AS months_crossed, + 2 AS hours_crossed, + 120 AS minutes_elapsed, + -1 AS days_backwards, + '2027-01-01'::DATE AS period_end_month +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_daily_callbacks.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_daily_callbacks.sql new file mode 100644 index 0000000000..0bab78aef8 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_daily_callbacks.sql @@ -0,0 +1,47 @@ +TEST (name "rpt_edge_daily_callbacks__counts_one_instant_sent_at_two_offsets_once"); + +WITH __ref__int_edge_callback_times AS ( + SELECT + v.callback_id, + NULL AS wall_time_utc, + NULL AS wall_time_as_zoned, + v.event_at_utc::TIMESTAMP_NTZ AS event_at_utc, + NULL AS event_at_merchant, + v.event_local_date::DATE AS event_local_date + FROM VALUES + ('CB-01', '2026-03-01 04:15:00', '2026-03-01'), + ('CB-02', '2026-03-01 04:15:00', '2026-03-01'), + ('CB-03', '2026-03-01 04:15:00', '2026-03-01'), + ('CB-04', '2026-03-01 17:45:00', '2026-03-01'), + ('CB-05', '2026-03-01 04:15:00', '2026-02-28') AS v(callback_id, event_at_utc, event_local_date) +), + +__ref__stg_edge_gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + v.event_at::TIMESTAMP_TZ AS event_at + FROM VALUES + ('CB-01', '2026-03-01 10:00:00 +05:45'), + ('CB-02', '2026-03-01 04:15:00 +00:00'), + ('CB-03', '2026-03-01 04:15:00 -00:00'), + ('CB-04', '2026-03-01 23:30:00 +05:45'), + ('CB-05', '2026-02-28 18:45:00 -09:30') AS v(callback_id, event_at) +), + +__expected__rpt_edge_daily_callbacks AS ( + SELECT + '2026-02-28'::DATE AS event_local_date, + 1 AS callbacks, + 1 AS distinct_instants, + 1 AS distinct_offsets, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS first_event_utc + UNION ALL + SELECT + '2026-03-01'::DATE AS event_local_date, + 4 AS callbacks, + 2 AS distinct_instants, + 2 AS distinct_offsets, + '2026-03-01 04:15:00'::TIMESTAMP_NTZ(9) AS first_event_utc +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_gateway_callbacks.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_gateway_callbacks.sql new file mode 100644 index 0000000000..3faa8b004a --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_gateway_callbacks.sql @@ -0,0 +1,295 @@ +TEST (name "stg_edge_gateway_callbacks__reads_free_text_times_in_every_lenient_spelling"); + +WITH __source__edge__gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + NULL AS event_at, + NULL AS event_at_ns, + v.event_time_text, + NULL AS business_date, + NULL AS period_start, + NULL AS period_end, + NULL AS body, + '2026-03-02 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('CB-01', '2026-03-01 10:15:00'), + ('CB-02', '2026-03-01T10:15:00'), + ('CB-03', '2026-03-01T10:15:00Z'), + ('CB-04', '2026-03-01 10:15:00 +05:45'), + ('CB-05', '2026-03-01T10:15:00-09:30'), + ('CB-06', '01-MAR-26 10:15'), + ('CB-07', '2026-03-01 24:00:00'), + ('CB-08', '2026-3-1 7:5:0'), + ('CB-09', ' 2026-03-01 10:15:00 '), + ('CB-10', '1772359200'), + ('CB-11', '1772359200000'), + ('CB-12', 'now'), + ('CB-13', '\u0662\u0660\u0662\u0666-\u0660\u0663-\u0660\u0661'), + ('CB-14', ''), + ('CB-15', 'n/a'), + ('CB-16', '1962-07-01 12:00:00.1234567'), + ('CB-17', '10:15'), + ('CB-18', '2026-03-01'), + ('CB-19', '31/12/2026'), + ('CB-20', '2026-02-30') AS v(callback_id, event_time_text) +), + +__expected__stg_edge_gateway_callbacks AS ( + SELECT + 'CB-01' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-03' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-04' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 +05:45'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-05' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 -09:30'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-06' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-07' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-08' AS callback_id, + '2026-03-01 07:05:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 07:05:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-09' AS callback_id, + '2026-03-01 10:15:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:15:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-10' AS callback_id, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:00:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + '10:00:00'::TIME(9) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-11' AS callback_id, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 10:00:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + '10:00:00'::TIME(9) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-12' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-13' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-14' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-15' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-16' AS callback_id, + '1962-07-01 12:00:00.1234567'::TIMESTAMP_NTZ(9) AS text_timestamp, + '1962-07-01 12:00:00.1234567 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '1962-07-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '1962-07-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-17' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + '10:15:00'::TIME(9) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-18' AS callback_id, + '2026-03-01 00:00:00'::TIMESTAMP_NTZ(9) AS text_timestamp, + '2026-03-01 00:00:00 Z'::TIMESTAMP_TZ(9) AS text_timestamp_tz, + '2026-03-01'::DATE AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + '2026-03-01'::DATE AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-19' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month + UNION ALL + SELECT + 'CB-20' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp, + CAST(NULL AS TIMESTAMP_TZ(9)) AS text_timestamp_tz, + CAST(NULL AS DATE) AS text_date, + CAST(NULL AS TIME(9)) AS text_time, + CAST(NULL AS DATE) AS text_date_cast, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS text_timestamp_day_month +) + +TEST (name "stg_edge_gateway_callbacks__reads_epoch_milliseconds_sent_as_numbers_and_text"); + +WITH __source__edge__gateway_callbacks AS ( + SELECT + v.callback_id, + NULL AS merchant_zone, + NULL AS local_wall_time, + NULL AS event_at, + NULL AS event_at_ns, + NULL AS event_time_text, + NULL AS business_date, + NULL AS period_start, + NULL AS period_end, + PARSE_JSON(v.body) AS body, + '2026-03-02 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('CB-01', '{"ts_ms": 1772359200000}'), + ('CB-02', '{"ts_ms": "1772359200000"}'), + ('CB-03', '{"ts_ms": 1772359200}'), + ('CB-04', '{"ts_ms": "1772359200"}'), + ('CB-05', '{"ts_ms": 1772359200123}'), + ('CB-06', '{"ts_ms": 1772359200123.0}'), + ('CB-07', '{"ts_ms": null}'), + ('CB-08', '{}') AS v(callback_id, body) +), + +__expected__stg_edge_gateway_callbacks AS ( + SELECT + 'CB-01' AS callback_id, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '58133-10-29 16:00:00'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-02' AS callback_id, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-03' AS callback_id, + '1970-01-21 12:19:19.2'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-04' AS callback_id, + '1970-01-21 12:19:19.2'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-05' AS callback_id, + '2026-03-01 10:00:00.123'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00.123'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '58133-10-29 16:02:03'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-06' AS callback_id, + '2026-03-01 10:00:00.123'::TIMESTAMP_NTZ(3) AS sent_at_from_ms, + '2026-03-01 10:00:00.123'::TIMESTAMP_NTZ(9) AS sent_at_from_text, + '58133-10-29 16:02:03'::TIMESTAMP_NTZ(9) AS sent_at_cast + UNION ALL + SELECT + 'CB-07' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(3)) AS sent_at_from_ms, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS sent_at_from_text, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS sent_at_cast + UNION ALL + SELECT + 'CB-08' AS callback_id, + CAST(NULL AS TIMESTAMP_NTZ(3)) AS sent_at_from_ms, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS sent_at_from_text, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS sent_at_cast +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_price_windows.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_price_windows.sql new file mode 100644 index 0000000000..ecc18888a4 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_price_windows.sql @@ -0,0 +1,54 @@ +TEST (name "stg_edge_price_windows__closes_open_windows_with_sentinel_timestamps"); + +WITH __source__edge__price_windows AS ( + SELECT + v.window_id, + v.sku, + v.price_minor, + v.valid_from::TIMESTAMP_NTZ(6) AS valid_from, + v.valid_to::TIMESTAMP_NTZ(6) AS valid_to, + '2026-03-02 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PW-1', 'MALLARD-01', 1299, NULL, NULL), + ('PW-2', 'TEAL-02', 899, '0001-01-01 00:00:00', '9999-12-31 23:59:59.999999'), + ('PW-3', 'EIDER-03', 2450, '2026-01-01 00:00:00', NULL), + ('PW-4', 'PINTAIL-04', 1575, '1969-12-31 23:59:59.999999', '1970-01-01 00:00:00.000001'), + ('PW-5', 'MERGANSER-05', 650, '2026-03-29 01:30:00', '9999-06-30 00:00:00') AS v(window_id, sku, price_minor, valid_from, valid_to) +), + +__expected__stg_edge_price_windows AS ( + SELECT + 'PW-1' AS window_id, + '0001-01-01 00:00:00'::TIMESTAMP_NTZ(6) AS valid_from, + '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS valid_to, + TRUE AS is_open_ended, + 3652058 AS window_days + UNION ALL + SELECT + 'PW-2' AS window_id, + '0001-01-01 00:00:00'::TIMESTAMP_NTZ(6) AS valid_from, + '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS valid_to, + TRUE AS is_open_ended, + 3652058 AS window_days + UNION ALL + SELECT + 'PW-3' AS window_id, + '2026-01-01 00:00:00'::TIMESTAMP_NTZ(6) AS valid_from, + '9999-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS valid_to, + TRUE AS is_open_ended, + 2912442 AS window_days + UNION ALL + SELECT + 'PW-4' AS window_id, + '1969-12-31 23:59:59.999999'::TIMESTAMP_NTZ(6) AS valid_from, + '1970-01-01 00:00:00.000001'::TIMESTAMP_NTZ(6) AS valid_to, + FALSE AS is_open_ended, + 1 AS window_days + UNION ALL + SELECT + 'PW-5' AS window_id, + '2026-03-29 01:30:00'::TIMESTAMP_NTZ(6) AS valid_from, + '9999-06-30 00:00:00'::TIMESTAMP_NTZ(6) AS valid_to, + TRUE AS is_open_ended, + 2912171 AS window_days +) From 4d8b66fd7bcec12ce436a93fa170db37e9660eff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:27:44 +0000 Subject: [PATCH 3/6] test(benchmarks): add DuckShop edge cases for ordering, numbers and text Add customer profile snapshots, order ledger lines and support ticket notes to the edge-case area, with models and Snowflake-captured unit tests for: NULL ordering under DESC in dedupe windows and rankings; NULL propagation through || and CONCAT beside CONCAT_WS; a select alias that shadows a raw column in QUALIFY; regular expressions with anchoring, occurrences, groups and flags, written escaped and dollar-quoted; backslash-escaped quotes, # separators and // comments; ROUND half-way values as NUMBER and FLOAT, decimal division and zero divisors, FLOAT rates cast to fixed scales, MOD of negative and fractional values, and 38-digit amounts; UNION ALL BY NAME with EXCLUDE and an untyped NULL branch; LAST_VALUE and LAG IGNORE NULLS under default frames; ties in LISTAGG, ARRAY_AGG, MAX_BY, MIN_BY and MODE; aggregates over empty and all-NULL groups; number spellings in TRY_TO_NUMBER; names in many scripts through INITCAP, UPPER, LENGTH, LPAD, EDITDISTANCE and JAROWINKLER_SIMILARITY; ILIKE ANY, prefix NOT LIKE ANY and LIKE ESCAPE; NOT IN against a blocklist holding a NULL; FULL OUTER JOIN USING with NULL keys; integers past 2^53 against FLOAT bounds; PERCENTILE_CONT, PERCENTILE_DISC and MEDIAN; SPLIT_PART and SUBSTR at edge positions; and a cast that a filter must guard. --- .../int_edge_customer_eligibility.sql | 63 +++++ .../edge_cases/int_edge_customer_matches.sql | 42 +++ .../edge_cases/int_edge_line_amounts.sql | 72 +++++ .../edge_cases/int_edge_order_line_labels.sql | 58 ++++ .../edge_cases/int_edge_order_status.sql | 34 +++ .../int_edge_orders_all_channels.sql | 94 +++++++ .../edge_cases/int_edge_price_changes.sql | 67 +++++ .../edge_cases/int_edge_quantity_checks.sql | 44 +++ .../edge_cases/rpt_edge_channel_reasons.sql | 55 ++++ .../rpt_edge_stock_reconciliation.sql | 64 +++++ .../edge_cases/stg_edge_customer_profiles.sql | 74 +++++ .../edge_cases/stg_edge_ledger_entries.sql | 58 ++++ .../edge_cases/stg_edge_ticket_notes.sql | 72 +++++ .../project/python/loaders/_edge_cases.py | 113 ++++++++ .../project/python/loaders/edge_cases.py | 15 + .../duckshop/project/sources/edge_cases.yml | 116 ++++++++ .../duckshop/project/sqlbuild_project.toml | 8 + .../test_int_edge_customer_eligibility.sql | 88 ++++++ .../test_int_edge_customer_matches.sql | 71 +++++ .../edge_cases/test_int_edge_line_amounts.sql | 229 ++++++++++++++++ .../test_int_edge_order_line_labels.sql | 118 ++++++++ .../edge_cases/test_int_edge_order_status.sql | 51 ++++ .../test_int_edge_orders_all_channels.sql | 65 +++++ .../test_int_edge_price_changes.sql | 93 +++++++ .../test_int_edge_quantity_checks.sql | 59 ++++ .../test_rpt_edge_channel_reasons.sql | 151 ++++++++++ .../test_rpt_edge_stock_reconciliation.sql | 65 +++++ .../test_stg_edge_customer_profiles.sql | 254 +++++++++++++++++ .../test_stg_edge_ledger_entries.sql | 172 ++++++++++++ .../edge_cases/test_stg_edge_ticket_notes.sql | 257 ++++++++++++++++++ 30 files changed, 2722 insertions(+) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_eligibility.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_matches.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_line_amounts.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_line_labels.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_status.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_orders_all_channels.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_price_changes.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_quantity_checks.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_channel_reasons.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_stock_reconciliation.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_customer_profiles.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ledger_entries.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_eligibility.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_matches.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_line_amounts.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_status.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_orders_all_channels.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_price_changes.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_quantity_checks.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_channel_reasons.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_stock_reconciliation.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_customer_profiles.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ledger_entries.sql create mode 100644 benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ticket_notes.sql diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_eligibility.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_eligibility.sql new file mode 100644 index 0000000000..1261343c31 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_eligibility.sql @@ -0,0 +1,63 @@ +MODEL ( + description """ + One row per customer with whether a blocklist excludes them, checked with NOT IN against a + blocklist that may hold a missing customer id, and with NOT EXISTS + """, + tags [edge_case], + columns ( + customer_id (type VARCHAR, nullable false, audits [not_null, unique]), + passes_not_in (type BOOLEAN), + passes_not_exists (type BOOLEAN), + ), +); + +WITH profiles AS ( + SELECT p.customer_id + FROM __ref("stg_edge_customer_profiles") AS p +), + +blocked AS ( + SELECT s.customer_id + FROM __source("edge__customer_profiles") AS s + WHERE s.loyalty_tier = 'BLOCKED' +), + +passes_not_in AS ( + SELECT p.customer_id + FROM profiles AS p + WHERE + p.customer_id NOT IN ( + SELECT b.customer_id + FROM blocked AS b + ) +), + +passes_not_exists AS ( + SELECT p.customer_id + FROM profiles AS p + WHERE + NOT EXISTS( + SELECT 1 + FROM blocked AS b + WHERE b.customer_id = p.customer_id + ) +), + +final AS ( + SELECT + p.customer_id, + ( + i.customer_id IS NOT NULL + ) AS passes_not_in, + ( + e.customer_id IS NOT NULL + ) AS passes_not_exists + FROM profiles AS p + LEFT JOIN passes_not_in AS i + ON p.customer_id = i.customer_id + LEFT JOIN passes_not_exists AS e + ON p.customer_id = e.customer_id +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_matches.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_matches.sql new file mode 100644 index 0000000000..8b9c54770e --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_customer_matches.sql @@ -0,0 +1,42 @@ +MODEL ( + description """ + One row per customer comparing the current and previous name for identity matching, by edit + distance and Jaro-Winkler similarity over characters, not bytes, and ranking partner order + numbers exactly and as FLOAT, where numbers past 2^53 tie + """, + tags [edge_case], + columns ( + customer_id (type VARCHAR, nullable false, audits [not_null, unique]), + name_edit_distance (type INT), + name_similarity (type INT), + is_same_person (type BOOLEAN), + order_number_rank (type INT), + order_number_float_rank (type INT), + ), +); + +WITH profiles AS ( + SELECT + p.customer_id, + p.full_name, + p.previous_full_name, + p.partner_order_number + FROM __ref("stg_edge_customer_profiles") AS p +), + +final AS ( + SELECT + p.customer_id, + EDITDISTANCE(LOWER(p.full_name), LOWER(p.previous_full_name))::INT AS name_edit_distance, + JAROWINKLER_SIMILARITY(p.full_name, p.previous_full_name)::INT AS name_similarity, + COALESCE( + JAROWINKLER_SIMILARITY(p.full_name, p.previous_full_name) >= 90, + FALSE + ) AS is_same_person, + DENSE_RANK() OVER (ORDER BY p.partner_order_number DESC)::INT AS order_number_rank, + DENSE_RANK() OVER (ORDER BY p.partner_order_number::FLOAT DESC)::INT AS order_number_float_rank + FROM profiles AS p +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_line_amounts.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_line_amounts.sql new file mode 100644 index 0000000000..4675f3109d --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_line_amounts.sql @@ -0,0 +1,72 @@ +MODEL ( + description """ + One row per order ledger line with money arithmetic at its edges: decimal division and zero + divisors, rounding half-way values as NUMBER and FLOAT, FLOAT exchange rates cast to fixed + scales, MOD bucketing of negative and fractional values, and 38-digit amounts + """, + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + unit_amount (type NUMBER(38, 10)), + share_of_three (type NUMBER(38, 10)), + unit_amount_div0 (type NUMBER(38, 10)), + unit_amount_nullifzero (type NUMBER(38, 10)), + amount_rounded (type NUMBER(38, 2)), + amount_rounded_whole (type NUMBER(38, 0)), + amount_float_rounded (type FLOAT), + converted_minor (type FLOAT), + converted_2dp (type NUMBER(18, 2)), + converted_6dp (type NUMBER(18, 6)), + converted_rounded (type NUMBER(18, 2)), + line_bucket (type NUMBER(38, 10)), + rate_bucket (type FLOAT), + amount_halved (type NUMBER(38, 10)), + ), +); + +WITH lines AS ( + SELECT + e.entry_id, + e.quantity, + e.amount, + e.unit_price_minor, + e.fx_rate + FROM __ref("stg_edge_ledger_entries") AS e +), + +final AS ( + SELECT + l.entry_id, + ( + l.amount / NULLIF(l.quantity, 0) + )::NUMBER(38, 10) AS unit_amount, + ( + l.amount / 3 + )::NUMBER(38, 10) AS share_of_three, + DIV0(l.amount, l.quantity)::NUMBER(38, 10) AS unit_amount_div0, + ( + l.amount / NULLIFZERO(l.quantity) + )::NUMBER(38, 10) AS unit_amount_nullifzero, + ROUND(l.amount, 2)::NUMBER(38, 2) AS amount_rounded, + ROUND(l.amount)::NUMBER(38, 0) AS amount_rounded_whole, + ROUND(l.amount::FLOAT)::FLOAT AS amount_float_rounded, + ( + l.unit_price_minor * l.fx_rate + )::FLOAT AS converted_minor, + ( + l.unit_price_minor * l.fx_rate + )::NUMBER(18, 2) AS converted_2dp, + ( + l.unit_price_minor * l.fx_rate + )::NUMBER(18, 6) AS converted_6dp, + ROUND(l.unit_price_minor * l.fx_rate, 2)::NUMBER(18, 2) AS converted_rounded, + MOD(l.amount, 7)::NUMBER(38, 10) AS line_bucket, + MOD(l.fx_rate, 0.5)::FLOAT AS rate_bucket, + ( + l.amount / 2 + )::NUMBER(38, 10) AS amount_halved + FROM lines AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_line_labels.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_line_labels.sql new file mode 100644 index 0000000000..c20bca73ff --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_line_labels.sql @@ -0,0 +1,58 @@ +MODEL ( + description """ + One row per order with its lines listed in a total order, the priciest and cheapest SKU, the + customer tier taken from any line, and percentiles, medians and modes of its unit prices + """, + tags [edge_case], + columns ( + order_ref (type VARCHAR, nullable false, audits [not_null, unique]), + line_count (type INT), + sku_list (type VARCHAR), + sku_array (type ARRAY), + priciest_sku (type VARCHAR), + cheapest_sku (type VARCHAR), + customer_tier (type VARCHAR), + median_unit_price (type NUMBER(38, 3)), + percentile_unit_price (type NUMBER(38, 3)), + percentile_disc_unit_price (type NUMBER(38, 0)), + modal_quantity (type NUMBER(38, 0)), + ), +); + +WITH lines AS ( + SELECT + e.entry_id, + e.order_ref, + e.line_no, + e.sku, + e.quantity, + e.unit_price_minor, + e.customer_tier + FROM __ref("stg_edge_ledger_entries") AS e + WHERE e.order_ref IS NOT NULL +), + +final AS ( + SELECT + l.order_ref, + COUNT(*)::INT AS line_count, + LISTAGG(l.sku, ',') WITHIN GROUP (ORDER BY l.line_no, l.sku, l.entry_id)::VARCHAR AS sku_list, + ARRAY_AGG(l.sku) WITHIN GROUP (ORDER BY l.line_no, l.sku, l.entry_id)::ARRAY AS sku_array, + MAX_BY(l.sku, l.unit_price_minor)::VARCHAR AS priciest_sku, + MIN_BY(l.sku, l.unit_price_minor)::VARCHAR AS cheapest_sku, + ANY_VALUE(l.customer_tier)::VARCHAR AS customer_tier, + MEDIAN(l.unit_price_minor)::NUMBER(38, 3) AS median_unit_price, + PERCENTILE_CONT(0.25) WITHIN GROUP ( + ORDER BY l.unit_price_minor + )::NUMBER(38, 3) AS percentile_unit_price, + PERCENTILE_DISC(0.25) WITHIN GROUP ( + ORDER BY l.unit_price_minor + )::NUMBER(38, 0) AS percentile_disc_unit_price, + MODE(l.quantity)::NUMBER(38, 0) AS modal_quantity + FROM lines AS l + GROUP BY + l.order_ref +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_status.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_status.sql new file mode 100644 index 0000000000..d24b8552db --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_order_status.sql @@ -0,0 +1,34 @@ +MODEL ( + description """ + One row per order with its status upper-cased, preferring a shipped ledger line, where the + window ordering names the raw status column that the upper-cased select alias shadows + """, + tags [edge_case], + columns ( + order_ref (type VARCHAR, nullable false, audits [not_null, unique]), + status (type VARCHAR), + entry_id (type VARCHAR), + ), +); + +WITH lines AS ( + SELECT + e.entry_id, + e.order_ref, + e.status + FROM __ref("stg_edge_ledger_entries") AS e + WHERE e.order_ref IS NOT NULL +), + +final AS ( + SELECT + l.order_ref, + UPPER(l.status)::VARCHAR AS status, + l.entry_id + FROM lines AS l + QUALIFY + ROW_NUMBER() OVER (PARTITION BY l.order_ref ORDER BY status = 'SHIPPED' DESC, l.entry_id) = 1 +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_orders_all_channels.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_orders_all_channels.sql new file mode 100644 index 0000000000..66e27e614a --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_orders_all_channels.sql @@ -0,0 +1,94 @@ +MODEL ( + description """ + One row per ledger line across the web, app and partner channels, unioned by column name where + each channel lists its columns in its own order, the partner channel has no rate and an untyped + missing refund reason + """, + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + channel (type VARCHAR), + order_ref (type VARCHAR), + quantity (type NUMBER(38, 0)), + amount (type NUMBER(38, 10)), + refund_reason (type VARCHAR), + rate (type NUMBER(38, 10)), + ), +); + +WITH lines AS ( + SELECT + e.entry_id, + e.order_ref, + e.channel, + e.quantity, + e.amount, + e.reason_code, + e.fx_rate + FROM __ref("stg_edge_ledger_entries") AS e +), + +web AS ( + SELECT + l.entry_id, + l.channel, + l.order_ref, + l.quantity, + l.amount, + l.reason_code AS refund_reason, + 1::NUMBER(10, 5) AS rate + FROM lines AS l + WHERE l.channel = 'web' +), + +app AS ( + SELECT + l.amount, + l.quantity, + l.reason_code AS refund_reason, + l.order_ref, + l.channel, + l.entry_id, + l.fx_rate::NUMBER(10, 5) AS rate + FROM lines AS l + WHERE l.channel = 'app' +), + +partner AS ( + SELECT + l.entry_id, + l.order_ref, + l.channel, + l.quantity, + l.amount, + NULL AS refund_reason, + l.quantity::NUMBER(38, 0) AS rate + FROM lines AS l + WHERE l.channel = 'partner' +), + +unioned AS ( + SELECT * + FROM web + UNION ALL BY NAME + SELECT * + FROM app + UNION ALL BY NAME + SELECT * EXCLUDE (rate) + FROM partner +), + +final AS ( + SELECT + u.entry_id, + u.channel, + u.order_ref, + u.quantity, + u.amount, + u.refund_reason::VARCHAR AS refund_reason, + u.rate::NUMBER(38, 10) AS rate + FROM unioned AS u +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_price_changes.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_price_changes.sql new file mode 100644 index 0000000000..b9eeebeb09 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_price_changes.sql @@ -0,0 +1,67 @@ +MODEL ( + description """ + One row per ledger price change of a SKU with the price before it, skipping missing prices, the + first and running last price under the default window frame, and its rank among the SKU's + changes with the latest first, where a change without a time sorts first + """, + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + sku (type VARCHAR), + unit_price_minor (type NUMBER(38, 0)), + previous_price (type NUMBER(38, 0)), + previous_known_price (type NUMBER(38, 0)), + first_price (type NUMBER(38, 0)), + running_last_price (type NUMBER(38, 0)), + partition_last_price (type NUMBER(38, 0)), + recency_rank (type INT), + price_rank (type INT), + ), +); + +WITH changes AS ( + SELECT + e.entry_id, + e.sku, + e.unit_price_minor, + e.changed_at + FROM __ref("stg_edge_ledger_entries") AS e + WHERE e.sku IS NOT NULL +), + +final AS ( + SELECT + c.entry_id, + c.sku, + c.unit_price_minor, + LAG(c.unit_price_minor) OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at, c.entry_id + )::NUMBER(38, 0) AS previous_price, + LAG(c.unit_price_minor) IGNORE NULLS OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at, c.entry_id + )::NUMBER(38, 0) AS previous_known_price, + FIRST_VALUE(c.unit_price_minor) OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at, c.entry_id + )::NUMBER(38, 0) AS first_price, + LAST_VALUE(c.unit_price_minor) OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at, c.entry_id + )::NUMBER(38, 0) AS running_last_price, + LAST_VALUE(c.unit_price_minor) OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at, c.entry_id + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + )::NUMBER(38, 0) AS partition_last_price, + ROW_NUMBER() OVER ( + PARTITION BY c.sku + ORDER BY c.changed_at DESC, c.entry_id DESC + )::INT AS recency_rank, + DENSE_RANK() OVER (PARTITION BY c.sku ORDER BY c.unit_price_minor DESC)::INT AS price_rank + FROM changes AS c +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_quantity_checks.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_quantity_checks.sql new file mode 100644 index 0000000000..2fd4e0af6b --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_quantity_checks.sql @@ -0,0 +1,44 @@ +MODEL ( + description """ + One row per ledger line whose quantity text is numeric, comparing text and numbers with text + literals, and casting the text only after a filter has removed text that is not a number + """, + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + quantity_text (type VARCHAR), + text_equals_seven (type BOOLEAN), + quantity_equals_seven_text (type BOOLEAN), + quantity_cast (type NUMBER(18, 0)), + ), +); + +WITH lines AS ( + SELECT + e.entry_id, + e.quantity, + e.quantity_text + FROM __ref("stg_edge_ledger_entries") AS e +), + +numeric_lines AS ( + SELECT + l.entry_id, + l.quantity, + l.quantity_text + FROM lines AS l + WHERE TRY_TO_NUMBER(l.quantity_text) IS NOT NULL +), + +final AS ( + SELECT + n.entry_id, + n.quantity_text, + n.quantity_text = '7' AS text_equals_seven, + n.quantity = '7' AS quantity_equals_seven_text, + n.quantity_text::NUMBER(18, 0) AS quantity_cast + FROM numeric_lines AS n +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_channel_reasons.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_channel_reasons.sql new file mode 100644 index 0000000000..252723e0e4 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_channel_reasons.sql @@ -0,0 +1,55 @@ +MODEL ( + description """ + One row per sales channel, including channels with no ledger lines and channels whose lines + carry no reason code, with every aggregate over those empty and all-missing groups + """, + tags [edge_case], + columns ( + channel (type VARCHAR, nullable false, audits [not_null, unique]), + lines (type INT), + reasons (type INT), + refunded_lines (type INT), + any_refund (type BOOLEAN), + total_quantity (type NUMBER(38, 0)), + max_reason (type VARCHAR), + reason_list (type VARCHAR), + reason_array (type ARRAY), + reasons_by_entry (type OBJECT), + ), +); + +WITH channels AS ( + SELECT v.channel + FROM VALUES ('web'), ('app'), ('partner'), ('phone') AS v(channel) +), + +lines AS ( + SELECT + e.entry_id, + e.channel, + e.quantity, + e.reason_code + FROM __ref("stg_edge_ledger_entries") AS e +), + +final AS ( + SELECT + c.channel::VARCHAR AS channel, + COUNT(l.entry_id)::INT AS lines, + COUNT(l.reason_code)::INT AS reasons, + COUNT_IF(l.reason_code = 'REFUND')::INT AS refunded_lines, + BOOLOR_AGG(l.reason_code = 'REFUND')::BOOLEAN AS any_refund, + SUM(l.quantity)::NUMBER(38, 0) AS total_quantity, + MAX(l.reason_code)::VARCHAR AS max_reason, + LISTAGG(l.reason_code, ',') WITHIN GROUP (ORDER BY l.entry_id)::VARCHAR AS reason_list, + ARRAY_AGG(l.reason_code) WITHIN GROUP (ORDER BY l.entry_id)::ARRAY AS reason_array, + OBJECT_AGG(l.entry_id, l.reason_code::VARIANT)::OBJECT AS reasons_by_entry + FROM channels AS c + LEFT JOIN lines AS l + ON c.channel = l.channel + GROUP BY + c.channel +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_stock_reconciliation.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_stock_reconciliation.sql new file mode 100644 index 0000000000..23730deb11 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_stock_reconciliation.sql @@ -0,0 +1,64 @@ +MODEL ( + description """ + One row per SKU and warehouse that is allocated or counted, reconciling the two with a full + outer join on shared key columns where either side can be missing + """, + tags [edge_case], + columns ( + sku (type VARCHAR), + warehouse_code (type VARCHAR), + allocated_units (type NUMBER(38, 0)), + counted_units (type NUMBER(38, 0)), + variance_units (type NUMBER(38, 0), audits [not_null]), + ), +); + +WITH lines AS ( + SELECT + e.sku, + e.warehouse_code, + e.quantity, + e.status + FROM __ref("stg_edge_ledger_entries") AS e +), + +allocated AS ( + SELECT + l.sku, + l.warehouse_code, + SUM(l.quantity) AS allocated_units + FROM lines AS l + WHERE UPPER(l.status) = 'ALLOCATED' + GROUP BY + l.sku, + l.warehouse_code +), + +counted AS ( + SELECT + l.sku, + l.warehouse_code, + SUM(l.quantity) AS counted_units + FROM lines AS l + WHERE UPPER(l.status) = 'COUNTED' + GROUP BY + l.sku, + l.warehouse_code +), + +final AS ( + SELECT + sku, + warehouse_code, + a.allocated_units::NUMBER(38, 0) AS allocated_units, + c.counted_units::NUMBER(38, 0) AS counted_units, + ( + COALESCE(c.counted_units, 0) - COALESCE(a.allocated_units, 0) + )::NUMBER(38, 0) AS variance_units + FROM allocated AS a + FULL OUTER JOIN counted AS c + USING (sku, warehouse_code) +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_customer_profiles.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_customer_profiles.sql new file mode 100644 index 0000000000..a1eb1efdaa --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_customer_profiles.sql @@ -0,0 +1,74 @@ +MODEL ( + description """ + One row per customer from the most recently updated profile snapshot, where a snapshot without + an update time sorts first, with display names and addresses joined from parts that may be + missing, and names in many scripts measured and re-cased + """, + tags [edge_case], + columns ( + customer_id (type VARCHAR, nullable false, audits [not_null, unique]), + profile_id (type VARCHAR), + profile_updated_at (type TIMESTAMP_NTZ), + loyalty_tier (type VARCHAR), + partner_order_number (type NUMBER(38, 0)), + partner_order_number_float (type FLOAT), + is_long_order_number (type BOOLEAN), + full_name (type VARCHAR), + previous_full_name (type VARCHAR), + display_name (type VARCHAR), + display_name_concat (type VARCHAR), + display_name_skip_nulls (type VARCHAR), + postal_address (type VARCHAR), + name_title_case (type VARCHAR), + name_upper (type VARCHAR), + name_length (type INT), + name_initial (type VARCHAR), + first_name_padded (type VARCHAR), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__customer_profiles") +), + +latest AS ( + SELECT * + FROM src AS s + WHERE s.customer_id IS NOT NULL + QUALIFY + ROW_NUMBER() OVER ( + PARTITION BY s.customer_id + ORDER BY s.profile_updated_at DESC, s.profile_id DESC + ) = 1 +), + +final AS ( + SELECT + l.customer_id, + l.profile_id, + l.profile_updated_at, + l.loyalty_tier, + l.partner_order_number, + l.partner_order_number::FLOAT AS partner_order_number_float, + l.partner_order_number BETWEEN 9007199254740992::FLOAT AND 1e19::FLOAT AS is_long_order_number, + l.full_name, + l.previous_full_name, + ( + l.first_name || ' ' || l.middle_name || ' ' || l.last_name + )::VARCHAR AS display_name, + CONCAT(l.first_name, ' ', l.last_name)::VARCHAR AS display_name_concat, + CONCAT_WS(' ', l.first_name, l.middle_name, l.last_name)::VARCHAR AS display_name_skip_nulls, + ( + l.address_line1 || ', ' || l.address_line2 + )::VARCHAR AS postal_address, + INITCAP(l.full_name)::VARCHAR AS name_title_case, + UPPER(l.full_name)::VARCHAR AS name_upper, + LENGTH(l.full_name)::INT AS name_length, + LEFT(l.full_name, 1)::VARCHAR AS name_initial, + LPAD(l.first_name, 8, '*')::VARCHAR AS first_name_padded + FROM latest AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ledger_entries.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ledger_entries.sql new file mode 100644 index 0000000000..8b029e9666 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ledger_entries.sql @@ -0,0 +1,58 @@ +MODEL ( + description """ + One row per order ledger line, with the quantity text read in every number spelling and the + status kept as sent + """, + tags [edge_case], + columns ( + entry_id (type VARCHAR, nullable false, audits [not_null, unique]), + order_ref (type VARCHAR), + channel (type VARCHAR), + line_no (type NUMBER(38, 0)), + sku (type VARCHAR), + warehouse_code (type VARCHAR), + quantity (type NUMBER(38, 0)), + quantity_text (type VARCHAR), + quantity_from_text (type NUMBER(18, 2)), + quantity_text_decimal (type NUMBER(18, 0)), + unit_price_minor (type NUMBER(38, 0)), + amount (type NUMBER(38, 10)), + currency_code (type VARCHAR), + fx_rate (type FLOAT), + status (type VARCHAR), + reason_code (type VARCHAR), + customer_tier (type VARCHAR), + changed_at (type TIMESTAMP_NTZ), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__ledger_entries") +), + +final AS ( + SELECT + s.entry_id, + s.order_ref, + s.channel, + s.line_no, + s.sku, + s.warehouse_code, + s.quantity, + s.quantity_text, + TRY_TO_NUMBER(s.quantity_text, 18, 2)::NUMBER(18, 2) AS quantity_from_text, + TRY_TO_DECIMAL(s.quantity_text)::NUMBER(18, 0) AS quantity_text_decimal, + s.unit_price_minor, + s.amount, + s.currency_code, + s.fx_rate, + s.status, + s.reason_code, + s.customer_tier, + s.changed_at + FROM src AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql new file mode 100644 index 0000000000..d187342865 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_ticket_notes.sql @@ -0,0 +1,72 @@ +MODEL ( + description """ + One row per support ticket note with order references and reasons found by regular expressions, + written with escaped and dollar-quoted patterns, campaign and discount codes matched by pattern + lists and escapes, and SKUs split at edge positions + """, + tags [edge_case], + columns ( + note_id (type VARCHAR, nullable false, audits [not_null, unique]), + first_order_ref (type VARCHAR), + first_order_ref_dollar (type VARCHAR), + second_order_number (type VARCHAR), + mentions_order_rlike (type BOOLEAN), + mentions_order_regexp_like (type BOOLEAN), + order_ref_count (type INT), + refund_word (type VARCHAR), + body_collapsed (type VARCHAR), + quotes_customer (type BOOLEAN), + second_hash_tag (type VARCHAR), + is_newsletter_visit (type BOOLEAN), + is_customer_code (type BOOLEAN), + is_fifty_percent_code (type BOOLEAN), + sku_part_zero (type VARCHAR), + sku_part_last (type VARCHAR), + sku_part_beyond (type VARCHAR), + sku_part_no_delimiter (type VARCHAR), + sku_from_zero (type VARCHAR), + sku_last_two (type VARCHAR), + sku_before_start (type VARCHAR), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__ticket_notes") +), + +final AS ( + SELECT + s.note_id, + REGEXP_SUBSTR(s.body, 'DS-O-\\d+')::VARCHAR AS first_order_ref, + REGEXP_SUBSTR(s.body, $$DS-O-\d+$$)::VARCHAR AS first_order_ref_dollar, + REGEXP_SUBSTR(s.body, 'DS-O-(\\d+)', 1, 2, 'e', 1)::VARCHAR AS second_order_number, + ( + s.body RLIKE 'DS-O-\\d+' + )::BOOLEAN AS mentions_order_rlike, + REGEXP_LIKE(s.body, '.*ds-o-\\d+.*', 'is') AS mentions_order_regexp_like, + REGEXP_COUNT(s.body, 'DS-O-\\d+')::INT AS order_ref_count, + REGEXP_SUBSTR(s.body, 'refund(ed)?', 1, 1, 'i')::VARCHAR AS refund_word, + REGEXP_REPLACE(s.body, '\\s+', ' ')::VARCHAR AS body_collapsed, + // A backslash-escaped quote inside a single-quoted literal + CONTAINS(s.body, 'O\'Brien')::BOOLEAN AS quotes_customer, + SPLIT_PART(s.tags_text, '#', 2)::VARCHAR AS second_hash_tag, + ( + s.utm_source ILIKE ANY ('%mail%', 'news%') + ) AS is_newsletter_visit, + ( + NOT s.discount_code LIKE ANY ('TEST%', 'STAFF%') + ) AS is_customer_code, + s.discount_code LIKE '50!%%' ESCAPE '!' AS is_fifty_percent_code, + SPLIT_PART(s.sku, '-', 0)::VARCHAR AS sku_part_zero, + SPLIT_PART(s.sku, '-', -1)::VARCHAR AS sku_part_last, + SPLIT_PART(s.sku, '-', 5)::VARCHAR AS sku_part_beyond, + SPLIT_PART(s.sku, '', 1)::VARCHAR AS sku_part_no_delimiter, + SUBSTR(s.sku, 0, 3)::VARCHAR AS sku_from_zero, + SUBSTR(s.sku, -2)::VARCHAR AS sku_last_two, + SUBSTR(s.sku, -10, 3)::VARCHAR AS sku_before_start + FROM src AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/python/loaders/_edge_cases.py b/benchmarks/duckshop/project/python/loaders/_edge_cases.py index 958f475634..697d7b5f4a 100644 --- a/benchmarks/duckshop/project/python/loaders/_edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/_edge_cases.py @@ -8,6 +8,7 @@ from __future__ import annotations from datetime import datetime +from decimal import Decimal Rows = list[dict[str, object]] @@ -174,8 +175,120 @@ def price_windows() -> Rows: ] +def customer_profiles() -> Rows: + profiles = [ + # customer, full name, previous name, first, middle, last, line 1, line 2, tier, number, updated + ("C-1", "Größe Ente", "Grosse Ente", "Jürgen", None, "Größe", "1 Pond Lane", None, + "Golden Duck", 12345678901234567890, "2026-03-01 09:00:00"), + ("C-1", "Größe Ente", "Grosse Ente", "Jürgen", None, "Größe", "1 Pond Lane", "Flat 2", + "Golden Duck", 12345678901234567890, None), + ("C-2", "e\u0301lodie o'neill-smith", "elodie oneill smith", "Élodie", "", "O'Neill", + "2 Marsh Road", "", "Duckling", 9007199254740993, "2026-03-02 10:00:00"), + ("C-3", "ijsselmeer van der eend", "ijsselmeer van der eend", "IJsbrand", "van der", "Eend", + None, None, "BLOCKED", 9007199254740992, "2026-03-03 11:00:00"), + (None, "ann (dup)", None, None, None, None, None, None, "BLOCKED", None, None), + ("C-4", "ნინო ბერიძე", "ნინო ბერიძე", "ნინო", None, "ბერიძე", "4 Nest Street", + "Unit 🦆", "Mallard", -9223372036854775809, "2026-03-04 12:00:00"), + ] + rows: Rows = [] + for index, profile in enumerate(profiles): + (customer, full, previous, first, middle, last, line1, line2, tier, number, updated) = profile + rows.append( + { + "profile_id": f"PR-{index + 1:04d}", + "customer_id": customer, + "full_name": full, + "previous_full_name": previous, + "first_name": first, + "middle_name": middle, + "last_name": last, + "address_line1": line1, + "address_line2": line2, + "loyalty_tier": tier, + "partner_order_number": number, + "profile_updated_at": updated, + "_loaded_at": _at("2026-03-05 00:00:00"), + } + ) + return rows + + +def ledger_entries() -> Rows: + entries = [ + # order, channel, line, sku, warehouse, qty, qty text, price, amount, currency, rate, + # status, reason, tier, changed at + ("DS-O-2001", "web", 1, "MALLARD-01", "POND", 3, "007", 1299, "2.5", "GBP", 1.27, + "shipped", None, "Golden Duck", "2026-03-01 09:00:00"), + ("DS-O-2001", "web", 1, "TEAL-02", "POND", 1, "+7", 1299, "-2.5", "GBP", 1.27, + "Shipped", "REFUND", "Golden Duck", "2026-03-01 09:00:00"), + ("DS-O-2001", "web", 2, "EIDER-03", "POND", 2, "1e0", 899, "0.005", "GBP", 0.1, + "allocated", None, "Golden Duck", None), + ("DS-O-2002", "app", 1, "MALLARD-01", "NEST", 0, " 1.5 ", 1350, "1.0000000005", "KWD", + 3.25, "counted", "DAMAGED", "Duckling", "2026-03-02 09:00:00"), + ("DS-O-2002", "app", 2, "PINTAIL-04", "NEST", -4, "-0", None, "-7.5", "JPY", 0.0067, + "ALLOCATED", None, "Duckling", "2026-03-02 09:00:00"), + ("DS-O-2003", "partner", 1, "TEAL-02", "MARSH", 5, "1,000", 1100, + "123456789012345678901234567.0123456789", "AUD", 0.66, "Counted", None, None, + "2026-03-03 09:00:00"), + ("DS-O-2003", "partner", 2, "TEAL-02", "MARSH", 5, "n/a", 1100, "0", "AUD", 0.66, + "pending", None, None, "2026-03-03 09:00:00"), + ] + rows: Rows = [] + for index, entry in enumerate(entries): + (order, channel, line, sku, warehouse, qty, qty_text, price, amount, currency, rate, + status, reason, tier, changed) = entry + rows.append( + { + "entry_id": f"LE-{index + 1:04d}", + "order_ref": order, + "channel": channel, + "line_no": line, + "sku": sku, + "warehouse_code": warehouse, + "quantity": qty, + "quantity_text": qty_text, + "unit_price_minor": price, + "amount": Decimal(amount), + "currency_code": currency, + "fx_rate": rate, + "status": status, + "reason_code": reason, + "customer_tier": tier, + "changed_at": changed, + "_loaded_at": _at("2026-03-05 00:00:00"), + } + ) + return rows + + +def ticket_notes() -> Rows: + notes = [ + ("Customer O'Brien asks about DS-O-1001 and DS-O-1002; refund\tplease", + "bath#gift#teal", "newsletter", "50%OFF", "MALLARD-01"), + ("Order ds-o-77 was REFUNDED.\nPath C:\\ducks\\pond # not a comment", + "#leading", "Email", "STAFF10", "DUCK--"), + ("No order here", "", "partner_feed", "50PERCENT", ""), + (None, None, None, None, "A-B-C-D"), + ] + return [ + { + "note_id": f"TN-{index + 1:04d}", + "body": body, + "tags_text": tags, + "utm_source": utm, + "discount_code": code, + "sku": sku, + "_loaded_at": _at("2026-03-05 00:00:00"), + } + for index, (body, tags, utm, code, sku) in enumerate(notes) + ] + + TABLES: dict[str, Rows] = { "edge__partner_payloads": partner_payloads(), "edge__gateway_callbacks": gateway_callbacks(), "edge__price_windows": price_windows(), + "edge__customer_profiles": customer_profiles(), + "edge__ledger_entries": ledger_entries(), + "edge__ticket_notes": ticket_notes(), } diff --git a/benchmarks/duckshop/project/python/loaders/edge_cases.py b/benchmarks/duckshop/project/python/loaders/edge_cases.py index 8f931ba966..cb8af07a79 100644 --- a/benchmarks/duckshop/project/python/loaders/edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/edge_cases.py @@ -31,3 +31,18 @@ def edge__gateway_callbacks(ctx: LoaderContext) -> object: @loader def edge__price_windows(ctx: LoaderContext) -> object: return _load(ctx, "edge__price_windows") + + +@loader +def edge__customer_profiles(ctx: LoaderContext) -> object: + return _load(ctx, "edge__customer_profiles") + + +@loader +def edge__ledger_entries(ctx: LoaderContext) -> object: + return _load(ctx, "edge__ledger_entries") + + +@loader +def edge__ticket_notes(ctx: LoaderContext) -> object: + return _load(ctx, "edge__ticket_notes") diff --git a/benchmarks/duckshop/project/sources/edge_cases.yml b/benchmarks/duckshop/project/sources/edge_cases.yml index e30a0f265b..6bb45d8c59 100644 --- a/benchmarks/duckshop/project/sources/edge_cases.yml +++ b/benchmarks/duckshop/project/sources/edge_cases.yml @@ -94,3 +94,119 @@ sources: nullable: false audits: - not_null + - name: edge__customer_profiles + description: >- + Append-only customer profile snapshots whose business update time is sometimes missing, with + names and addresses in many scripts and partner order numbers beyond double precision + managed: true + contract: enforced + columns: + - name: profile_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: customer_id + type: VARCHAR + - name: full_name + type: VARCHAR + - name: previous_full_name + type: VARCHAR + - name: first_name + type: VARCHAR + - name: middle_name + type: VARCHAR + - name: last_name + type: VARCHAR + - name: address_line1 + type: VARCHAR + - name: address_line2 + type: VARCHAR + - name: loyalty_tier + type: VARCHAR + - name: partner_order_number + type: NUMBER(38,0) + - name: profile_updated_at + type: TIMESTAMP_NTZ + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null + - name: edge__ledger_entries + description: >- + Append-only order ledger lines from every channel, with amounts at full NUMBER precision, + FLOAT exchange rates, quantities sent as text and mixed-case statuses + managed: true + contract: enforced + columns: + - name: entry_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: order_ref + type: VARCHAR + - name: channel + type: VARCHAR + - name: line_no + type: NUMBER(38,0) + - name: sku + type: VARCHAR + - name: warehouse_code + type: VARCHAR + - name: quantity + type: NUMBER(38,0) + - name: quantity_text + type: VARCHAR + - name: unit_price_minor + type: NUMBER(38,0) + - name: amount + type: NUMBER(38,10) + - name: currency_code + type: VARCHAR + - name: fx_rate + type: FLOAT + - name: status + type: VARCHAR + - name: reason_code + type: VARCHAR + - name: customer_tier + type: VARCHAR + - name: changed_at + type: TIMESTAMP_NTZ + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null + - name: edge__ticket_notes + description: >- + Append-only support ticket notes whose text holds order references, quotes, backslashes and + separators, with the campaign and discount code the customer arrived with + managed: true + contract: enforced + columns: + - name: note_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: body + type: VARCHAR + - name: tags_text + type: VARCHAR + - name: utm_source + type: VARCHAR + - name: discount_code + type: VARCHAR + - name: sku + type: VARCHAR + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 5ce062fd4f..5338300a25 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -269,3 +269,11 @@ reason = """ The KPI grid is every reporting day for every region and metric pair that has ever had a value, so days without activity report zero instead of disappearing. Both sides are small and bounded. """ + +[[rules.rule_ignores]] +rules = ["SQBRSQL027"] +paths = ["models/marts/edge_cases/rpt_edge_stock_reconciliation.sql"] +reason = """ +The edge case selects the key columns that FULL OUTER JOIN ... USING merges. They belong to neither +side, so qualifying them with either alias would drop the keys of rows found only on the other side. +""" diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_eligibility.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_eligibility.sql new file mode 100644 index 0000000000..c6d8492821 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_eligibility.sql @@ -0,0 +1,88 @@ +TEST (name "int_edge_customer_eligibility__not_in_fails_everyone_when_the_blocklist_has_a_null"); + +WITH __ref__stg_edge_customer_profiles AS ( + SELECT v.customer_id + FROM VALUES ('C-1'), ('C-2'), ('C-3') AS v(customer_id) +), + +__source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + NULL AS full_name, + NULL AS previous_full_name, + NULL AS first_name, + NULL AS middle_name, + NULL AS last_name, + NULL AS address_line1, + NULL AS address_line2, + v.loyalty_tier, + NULL AS partner_order_number, + NULL AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', 'Duckling'), + ('PR-2', 'C-3', 'BLOCKED'), + ('PR-3', NULL, 'BLOCKED') AS v(profile_id, customer_id, loyalty_tier) +), + +__expected__int_edge_customer_eligibility AS ( + SELECT + 'C-1' AS customer_id, + FALSE AS passes_not_in, + TRUE AS passes_not_exists + UNION ALL + SELECT + 'C-2' AS customer_id, + FALSE AS passes_not_in, + TRUE AS passes_not_exists + UNION ALL + SELECT + 'C-3' AS customer_id, + FALSE AS passes_not_in, + FALSE AS passes_not_exists +) + +TEST (name "int_edge_customer_eligibility__not_in_and_not_exists_agree_without_nulls"); + +WITH __ref__stg_edge_customer_profiles AS ( + SELECT v.customer_id + FROM VALUES ('C-1'), ('C-2'), ('C-3') AS v(customer_id) +), + +__source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + NULL AS full_name, + NULL AS previous_full_name, + NULL AS first_name, + NULL AS middle_name, + NULL AS last_name, + NULL AS address_line1, + NULL AS address_line2, + v.loyalty_tier, + NULL AS partner_order_number, + NULL AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', 'Duckling'), + ('PR-2', 'C-3', 'BLOCKED') AS v(profile_id, customer_id, loyalty_tier) +), + +__expected__int_edge_customer_eligibility AS ( + SELECT + 'C-1' AS customer_id, + TRUE AS passes_not_in, + TRUE AS passes_not_exists + UNION ALL + SELECT + 'C-2' AS customer_id, + TRUE AS passes_not_in, + TRUE AS passes_not_exists + UNION ALL + SELECT + 'C-3' AS customer_id, + FALSE AS passes_not_in, + FALSE AS passes_not_exists +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_matches.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_matches.sql new file mode 100644 index 0000000000..2f9af431fb --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_customer_matches.sql @@ -0,0 +1,71 @@ +TEST (name "int_edge_customer_matches__compares_names_by_characters_and_ranks_long_numbers"); + +WITH __ref__stg_edge_customer_profiles AS ( + SELECT + v.customer_id, + NULL AS profile_id, + NULL AS profile_updated_at, + NULL AS loyalty_tier, + v.partner_order_number::NUMBER(38, 0) AS partner_order_number, + NULL AS partner_order_number_float, + NULL AS is_long_order_number, + v.full_name, + v.previous_full_name + FROM VALUES + ('C-1', 'Größe Ente', 'Grosse Ente', '9007199254740992'), + ('C-2', 'e\u0301lodie o''neill', '\u00e9lodie o''neill', '9007199254740993'), + ('C-3', '\u0133sselmeer', 'ijsselmeer', '9007199254740991'), + ('C-4', 'Mallard 🦆', 'Mallard', '12'), + ('C-5', 'Teal', NULL, NULL), + ('C-6', 'Pintail', 'pintail', '12') AS v(customer_id, full_name, previous_full_name, partner_order_number) +), + +__expected__int_edge_customer_matches AS ( + SELECT + 'C-1' AS customer_id, + 3 AS name_edit_distance, + 87 AS name_similarity, + FALSE AS is_same_person, + 3 AS order_number_rank, + 2 AS order_number_float_rank + UNION ALL + SELECT + 'C-2' AS customer_id, + 2 AS name_edit_distance, + 80 AS name_similarity, + FALSE AS is_same_person, + 2 AS order_number_rank, + 2 AS order_number_float_rank + UNION ALL + SELECT + 'C-3' AS customer_id, + 2 AS name_edit_distance, + 89 AS name_similarity, + FALSE AS is_same_person, + 4 AS order_number_rank, + 3 AS order_number_float_rank + UNION ALL + SELECT + 'C-4' AS customer_id, + 2 AS name_edit_distance, + 95 AS name_similarity, + TRUE AS is_same_person, + 5 AS order_number_rank, + 4 AS order_number_float_rank + UNION ALL + SELECT + 'C-5' AS customer_id, + CAST(NULL AS NUMBER(38, 0)) AS name_edit_distance, + CAST(NULL AS NUMBER(38, 0)) AS name_similarity, + FALSE AS is_same_person, + 1 AS order_number_rank, + 1 AS order_number_float_rank + UNION ALL + SELECT + 'C-6' AS customer_id, + 0 AS name_edit_distance, + 100 AS name_similarity, + TRUE AS is_same_person, + 5 AS order_number_rank, + 4 AS order_number_float_rank +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_line_amounts.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_line_amounts.sql new file mode 100644 index 0000000000..54c2a92925 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_line_amounts.sql @@ -0,0 +1,229 @@ +TEST (name "int_edge_line_amounts__divides_rounds_and_buckets_money_at_its_edges"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + v.amount::NUMBER(38, 10) AS amount, + NULL AS currency_code, + v.fx_rate::FLOAT AS fx_rate + FROM VALUES + ('LE-01', 3, 1299, '2.5', '1.27'), + ('LE-02', 3, 1299, '-2.5', '1.27'), + ('LE-03', 2, 899, '0.005', '0.1'), + ('LE-04', 0, 1350, '1.0000000005', '3.25'), + ('LE-05', -4, 205, '-7.5', '0.0067'), + ('LE-06', NULL, 1005, '0.015', '1.005'), + ('LE-07', 7, 1, '-0.0000000001', '0.3'), + ('LE-08', 3, 100, '1', '-1.5') AS v(entry_id, quantity, unit_price_minor, amount, fx_rate) +), + +__expected__int_edge_line_amounts AS ( + SELECT + 'LE-01' AS entry_id, + 0.8333333333::NUMBER(38, 10) AS unit_amount, + 0.8333333333::NUMBER(38, 10) AS share_of_three, + 0.8333333333::NUMBER(38, 10) AS unit_amount_div0, + 0.8333333333::NUMBER(38, 10) AS unit_amount_nullifzero, + 2.50::NUMBER(38, 2) AS amount_rounded, + 3 AS amount_rounded_whole, + '3.0'::FLOAT AS amount_float_rounded, + '1649.73'::FLOAT AS converted_minor, + 1649.73::NUMBER(18, 2) AS converted_2dp, + 1649.730000::NUMBER(18, 6) AS converted_6dp, + 1649.73::NUMBER(18, 2) AS converted_rounded, + 2.5000000000::NUMBER(38, 10) AS line_bucket, + '0.27'::FLOAT AS rate_bucket, + 1.2500000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-02' AS entry_id, + -0.8333333333::NUMBER(38, 10) AS unit_amount, + -0.8333333333::NUMBER(38, 10) AS share_of_three, + -0.8333333333::NUMBER(38, 10) AS unit_amount_div0, + -0.8333333333::NUMBER(38, 10) AS unit_amount_nullifzero, + -2.50::NUMBER(38, 2) AS amount_rounded, + -3 AS amount_rounded_whole, + '-3.0'::FLOAT AS amount_float_rounded, + '1649.73'::FLOAT AS converted_minor, + 1649.73::NUMBER(18, 2) AS converted_2dp, + 1649.730000::NUMBER(18, 6) AS converted_6dp, + 1649.73::NUMBER(18, 2) AS converted_rounded, + -2.5000000000::NUMBER(38, 10) AS line_bucket, + '0.27'::FLOAT AS rate_bucket, + -1.2500000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-03' AS entry_id, + 0.0025000000::NUMBER(38, 10) AS unit_amount, + 0.0016666667::NUMBER(38, 10) AS share_of_three, + 0.0025000000::NUMBER(38, 10) AS unit_amount_div0, + 0.0025000000::NUMBER(38, 10) AS unit_amount_nullifzero, + 0.01::NUMBER(38, 2) AS amount_rounded, + 0 AS amount_rounded_whole, + '0.0'::FLOAT AS amount_float_rounded, + '89.9'::FLOAT AS converted_minor, + 89.90::NUMBER(18, 2) AS converted_2dp, + 89.900000::NUMBER(18, 6) AS converted_6dp, + 89.90::NUMBER(18, 2) AS converted_rounded, + 0.0050000000::NUMBER(38, 10) AS line_bucket, + '0.1'::FLOAT AS rate_bucket, + 0.0025000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-04' AS entry_id, + CAST(NULL AS NUMBER(38, 10)) AS unit_amount, + 0.3333333335::NUMBER(38, 10) AS share_of_three, + 0.0000000000::NUMBER(38, 10) AS unit_amount_div0, + CAST(NULL AS NUMBER(38, 10)) AS unit_amount_nullifzero, + 1.00::NUMBER(38, 2) AS amount_rounded, + 1 AS amount_rounded_whole, + '1.0'::FLOAT AS amount_float_rounded, + '4387.5'::FLOAT AS converted_minor, + 4387.50::NUMBER(18, 2) AS converted_2dp, + 4387.500000::NUMBER(18, 6) AS converted_6dp, + 4387.50::NUMBER(18, 2) AS converted_rounded, + 1.0000000005::NUMBER(38, 10) AS line_bucket, + '0.25'::FLOAT AS rate_bucket, + 0.5000000003::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-05' AS entry_id, + 1.8750000000::NUMBER(38, 10) AS unit_amount, + -2.5000000000::NUMBER(38, 10) AS share_of_three, + 1.8750000000::NUMBER(38, 10) AS unit_amount_div0, + 1.8750000000::NUMBER(38, 10) AS unit_amount_nullifzero, + -7.50::NUMBER(38, 2) AS amount_rounded, + -8 AS amount_rounded_whole, + '-8.0'::FLOAT AS amount_float_rounded, + '1.3735'::FLOAT AS converted_minor, + 1.37::NUMBER(18, 2) AS converted_2dp, + 1.373500::NUMBER(18, 6) AS converted_6dp, + 1.37::NUMBER(18, 2) AS converted_rounded, + -0.5000000000::NUMBER(38, 10) AS line_bucket, + '0.0067'::FLOAT AS rate_bucket, + -3.7500000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-06' AS entry_id, + CAST(NULL AS NUMBER(38, 10)) AS unit_amount, + 0.0050000000::NUMBER(38, 10) AS share_of_three, + CAST(NULL AS NUMBER(38, 10)) AS unit_amount_div0, + CAST(NULL AS NUMBER(38, 10)) AS unit_amount_nullifzero, + 0.02::NUMBER(38, 2) AS amount_rounded, + 0 AS amount_rounded_whole, + '0.0'::FLOAT AS amount_float_rounded, + '1010.0249999999999'::FLOAT AS converted_minor, + 1010.02::NUMBER(18, 2) AS converted_2dp, + 1010.025000::NUMBER(18, 6) AS converted_6dp, + 1010.02::NUMBER(18, 2) AS converted_rounded, + 0.0150000000::NUMBER(38, 10) AS line_bucket, + '0.004999999999999893'::FLOAT AS rate_bucket, + 0.0075000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-07' AS entry_id, + 0.0000000000::NUMBER(38, 10) AS unit_amount, + 0.0000000000::NUMBER(38, 10) AS share_of_three, + 0.0000000000::NUMBER(38, 10) AS unit_amount_div0, + 0.0000000000::NUMBER(38, 10) AS unit_amount_nullifzero, + 0.00::NUMBER(38, 2) AS amount_rounded, + 0 AS amount_rounded_whole, + '0.0'::FLOAT AS amount_float_rounded, + '0.3'::FLOAT AS converted_minor, + 0.30::NUMBER(18, 2) AS converted_2dp, + 0.300000::NUMBER(18, 6) AS converted_6dp, + 0.30::NUMBER(18, 2) AS converted_rounded, + -0.0000000001::NUMBER(38, 10) AS line_bucket, + '0.3'::FLOAT AS rate_bucket, + -0.0000000001::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-08' AS entry_id, + 0.3333333333::NUMBER(38, 10) AS unit_amount, + 0.3333333333::NUMBER(38, 10) AS share_of_three, + 0.3333333333::NUMBER(38, 10) AS unit_amount_div0, + 0.3333333333::NUMBER(38, 10) AS unit_amount_nullifzero, + 1.00::NUMBER(38, 2) AS amount_rounded, + 1 AS amount_rounded_whole, + '1.0'::FLOAT AS amount_float_rounded, + '-150.0'::FLOAT AS converted_minor, + -150.00::NUMBER(18, 2) AS converted_2dp, + -150.000000::NUMBER(18, 6) AS converted_6dp, + -150.00::NUMBER(18, 2) AS converted_rounded, + 1.0000000000::NUMBER(38, 10) AS line_bucket, + '0.0'::FLOAT AS rate_bucket, + 0.5000000000::NUMBER(38, 10) AS amount_halved +) + +TEST (name "int_edge_line_amounts__divides_36_digit_amounts_exactly"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + 1 AS unit_price_minor, + v.amount::NUMBER(38, 10) AS amount, + NULL AS currency_code, + 1.0::FLOAT AS fx_rate + FROM VALUES + ('LE-01', 3, '99999999999999999999999999.9999999999'), + ('LE-02', 7, '12345678901234567890123456.0123456789'), + ('LE-03', -3, '-99999999999999999999999999.9999999999'), + ('LE-04', 1, '0.0000000001') AS v(entry_id, quantity, amount) +), + +__expected__int_edge_line_amounts AS ( + SELECT + 'LE-01' AS entry_id, + 33333333333333333333333333.3333333333::NUMBER(38, 10) AS unit_amount, + 33333333333333333333333333.3333333333::NUMBER(38, 10) AS share_of_three, + 100000000000000000000000000.00::NUMBER(38, 2) AS amount_rounded, + 100000000000000000000000000 AS amount_rounded_whole, + 1.9999999999::NUMBER(38, 10) AS line_bucket, + 50000000000000000000000000.0000000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-02' AS entry_id, + 1763668414462081127160493.7160493827::NUMBER(38, 10) AS unit_amount, + 4115226300411522630041152.0041152263::NUMBER(38, 10) AS share_of_three, + 12345678901234567890123456.01::NUMBER(38, 2) AS amount_rounded, + 12345678901234567890123456 AS amount_rounded_whole, + 5.0123456789::NUMBER(38, 10) AS line_bucket, + 6172839450617283945061728.0061728395::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-03' AS entry_id, + 33333333333333333333333333.3333333333::NUMBER(38, 10) AS unit_amount, + -33333333333333333333333333.3333333333::NUMBER(38, 10) AS share_of_three, + -100000000000000000000000000.00::NUMBER(38, 2) AS amount_rounded, + -100000000000000000000000000 AS amount_rounded_whole, + -1.9999999999::NUMBER(38, 10) AS line_bucket, + -50000000000000000000000000.0000000000::NUMBER(38, 10) AS amount_halved + UNION ALL + SELECT + 'LE-04' AS entry_id, + 0.0000000001::NUMBER(38, 10) AS unit_amount, + 0.0000000000::NUMBER(38, 10) AS share_of_three, + 0.00::NUMBER(38, 2) AS amount_rounded, + 0 AS amount_rounded_whole, + 0.0000000001::NUMBER(38, 10) AS line_bucket, + 0.0000000001::NUMBER(38, 10) AS amount_halved +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql new file mode 100644 index 0000000000..b7bca8d240 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql @@ -0,0 +1,118 @@ +TEST (name "int_edge_order_line_labels__lists_lines_in_a_total_order_and_summarises_prices"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + v.order_ref, + NULL AS channel, + v.line_no::NUMBER(38, 0) AS line_no, + v.sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + NULL AS reason_code, + v.customer_tier + FROM VALUES + ('LE-01', 'DS-O-1', 2, 'TEAL-02', 1, 899, 'Mallard'), + ('LE-02', 'DS-O-1', 1, 'MALLARD-01', 2, 1299, 'Mallard'), + ('LE-03', 'DS-O-1', 1, 'EIDER-03', 2, 350, 'Mallard'), + ('LE-04', 'DS-O-2', 1, 'PINTAIL-04', 1, 100, NULL), + ('LE-05', 'DS-O-2', 2, 'PINTAIL-04', 3, 200, NULL), + ('LE-06', 'DS-O-2', 3, NULL, 3, 300, NULL), + ('LE-07', 'DS-O-2', 4, 'TEAL-02', 5, NULL, NULL), + ('LE-08', 'DS-O-3', 1, 'MALLARD-01', 1, 500, 'Duckling'), + ('LE-09', NULL, 1, 'TEAL-02', 1, 1, 'Duckling') AS v(entry_id, order_ref, line_no, sku, quantity, unit_price_minor, customer_tier) +), + +__expected__int_edge_order_line_labels AS ( + SELECT + 'DS-O-1' AS order_ref, + 3 AS line_count, + 'EIDER-03,MALLARD-01,TEAL-02' AS sku_list, + PARSE_JSON('["EIDER-03","MALLARD-01","TEAL-02"]')::ARRAY AS sku_array, + 'MALLARD-01' AS priciest_sku, + 'EIDER-03' AS cheapest_sku, + 'Mallard' AS customer_tier, + 899.000::NUMBER(38, 3) AS median_unit_price, + 624.500::NUMBER(38, 3) AS percentile_unit_price, + 350 AS percentile_disc_unit_price, + 2 AS modal_quantity + UNION ALL + SELECT + 'DS-O-2' AS order_ref, + 4 AS line_count, + 'PINTAIL-04,PINTAIL-04,TEAL-02' AS sku_list, + PARSE_JSON('["PINTAIL-04","PINTAIL-04","TEAL-02"]')::ARRAY AS sku_array, + CAST(NULL AS VARCHAR) AS priciest_sku, + 'PINTAIL-04' AS cheapest_sku, + CAST(NULL AS VARCHAR) AS customer_tier, + 200.000::NUMBER(38, 3) AS median_unit_price, + 150.000::NUMBER(38, 3) AS percentile_unit_price, + 100 AS percentile_disc_unit_price, + 3 AS modal_quantity + UNION ALL + SELECT + 'DS-O-3' AS order_ref, + 1 AS line_count, + 'MALLARD-01' AS sku_list, + PARSE_JSON('["MALLARD-01"]')::ARRAY AS sku_array, + 'MALLARD-01' AS priciest_sku, + 'MALLARD-01' AS cheapest_sku, + 'Duckling' AS customer_tier, + 500.000::NUMBER(38, 3) AS median_unit_price, + 500.000::NUMBER(38, 3) AS percentile_unit_price, + 500 AS percentile_disc_unit_price, + 1 AS modal_quantity +) + +TEST (name "int_edge_order_line_labels__picks_one_of_the_tied_lines_for_max_by_and_mode"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + v.order_ref, + NULL AS channel, + v.line_no::NUMBER(38, 0) AS line_no, + v.sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + NULL AS reason_code, + NULL AS customer_tier + FROM VALUES + ('LE-01', 'DS-O-1', 1, 'MALLARD-01', 1, 1299), + ('LE-02', 'DS-O-1', 2, 'TEAL-02', 2, 1299), + ('LE-03', 'DS-O-1', 3, 'EIDER-03', 1, 350), + ('LE-04', 'DS-O-1', 4, 'PINTAIL-04', 2, 350) AS v(entry_id, order_ref, line_no, sku, quantity, unit_price_minor) +), + +__expected__int_edge_order_line_labels AS ( + SELECT + 'DS-O-1' AS order_ref, + 4 AS line_count, + 'MALLARD-01,TEAL-02,EIDER-03,PINTAIL-04' AS sku_list, + 824.500::NUMBER(38, 3) AS median_unit_price +), + +__assert__tied_picks_come_from_the_tied_lines AS ( + SELECT l.order_ref + FROM __ref("int_edge_order_line_labels") AS l + WHERE + l.priciest_sku NOT IN ('MALLARD-01', 'TEAL-02') + OR l.cheapest_sku NOT IN ('EIDER-03', 'PINTAIL-04') + OR l.modal_quantity NOT IN (1, 2) +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_status.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_status.sql new file mode 100644 index 0000000000..02fdc3f816 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_status.sql @@ -0,0 +1,51 @@ +TEST (name "int_edge_order_status__orders_by_the_raw_status_not_the_upper_cased_alias"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + v.order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + NULL AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + NULL AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + v.status + FROM VALUES + ('LE-01', 'DS-O-1', 'pending'), + ('LE-02', 'DS-O-1', 'shipped'), + ('LE-03', 'DS-O-2', 'Pending'), + ('LE-04', 'DS-O-2', 'SHIPPED'), + ('LE-05', 'DS-O-3', 'pending'), + ('LE-06', 'DS-O-3', 'cancelled'), + ('LE-07', 'DS-O-4', NULL), + ('LE-08', 'DS-O-4', 'Shipped') AS v(entry_id, order_ref, status) +), + +__expected__int_edge_order_status AS ( + SELECT + 'DS-O-1' AS order_ref, + 'PENDING' AS status, + 'LE-01' AS entry_id + UNION ALL + SELECT + 'DS-O-2' AS order_ref, + 'SHIPPED' AS status, + 'LE-04' AS entry_id + UNION ALL + SELECT + 'DS-O-3' AS order_ref, + 'PENDING' AS status, + 'LE-05' AS entry_id + UNION ALL + SELECT + 'DS-O-4' AS order_ref, + CAST(NULL AS VARCHAR) AS status, + 'LE-07' AS entry_id +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_orders_all_channels.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_orders_all_channels.sql new file mode 100644 index 0000000000..aefc2abc4d --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_orders_all_channels.sql @@ -0,0 +1,65 @@ +TEST (name "int_edge_orders_all_channels__unions_channels_by_column_name"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + v.order_ref, + v.channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + NULL AS unit_price_minor, + v.amount::NUMBER(38, 10) AS amount, + NULL AS currency_code, + v.fx_rate::FLOAT AS fx_rate, + NULL AS status, + v.reason_code + FROM VALUES + ('LE-01', 'DS-O-1', 'web', 2, '25.98', 1.0, 'REFUND'), + ('LE-02', 'DS-O-2', 'app', 1, '8.99', 1.234567, NULL), + ('LE-03', 'DS-O-3', 'partner', 5, '55', 0.66, 'DAMAGED'), + ('LE-04', 'DS-O-4', 'phone', 1, '1', 1.0, NULL), + ('LE-05', 'DS-O-5', 'app', 3, '-2.5', 0.000005, 'REFUND') AS v(entry_id, order_ref, channel, quantity, amount, fx_rate, reason_code) +), + +__expected__int_edge_orders_all_channels AS ( + SELECT + 'LE-01' AS entry_id, + 'web' AS channel, + 'DS-O-1' AS order_ref, + 2 AS quantity, + 25.9800000000::NUMBER(38, 10) AS amount, + 'REFUND' AS refund_reason, + 1.0000000000::NUMBER(38, 10) AS rate + UNION ALL + SELECT + 'LE-02' AS entry_id, + 'app' AS channel, + 'DS-O-2' AS order_ref, + 1 AS quantity, + 8.9900000000::NUMBER(38, 10) AS amount, + CAST(NULL AS VARCHAR) AS refund_reason, + 1.2345700000::NUMBER(38, 10) AS rate + UNION ALL + SELECT + 'LE-03' AS entry_id, + 'partner' AS channel, + 'DS-O-3' AS order_ref, + 5 AS quantity, + 55.0000000000::NUMBER(38, 10) AS amount, + CAST(NULL AS VARCHAR) AS refund_reason, + CAST(NULL AS NUMBER(38, 10)) AS rate + UNION ALL + SELECT + 'LE-05' AS entry_id, + 'app' AS channel, + 'DS-O-5' AS order_ref, + 3 AS quantity, + -2.5000000000::NUMBER(38, 10) AS amount, + 'REFUND' AS refund_reason, + 0.0000100000::NUMBER(38, 10) AS rate +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_price_changes.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_price_changes.sql new file mode 100644 index 0000000000..d50c5db29d --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_price_changes.sql @@ -0,0 +1,93 @@ +TEST (name "int_edge_price_changes__reads_neighbouring_prices_under_default_frames"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + v.sku, + NULL AS warehouse_code, + NULL AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + NULL AS reason_code, + NULL AS customer_tier, + v.changed_at::TIMESTAMP_NTZ AS changed_at + FROM VALUES + ('LE-01', 'MALLARD-01', 1299, '2026-03-01 09:00:00'), + ('LE-02', 'MALLARD-01', NULL, '2026-03-02 09:00:00'), + ('LE-03', 'MALLARD-01', 1199, '2026-03-03 09:00:00'), + ('LE-04', 'MALLARD-01', 1199, '2026-03-03 09:00:00'), + ('LE-05', 'MALLARD-01', 1399, NULL), + ('LE-06', 'TEAL-02', 899, '2026-03-01 09:00:00'), + ('LE-07', NULL, 1, '2026-03-01 09:00:00') AS v(entry_id, sku, unit_price_minor, changed_at) +), + +__expected__int_edge_price_changes AS ( + SELECT + 'LE-01' AS entry_id, + CAST(NULL AS NUMBER(38, 0)) AS previous_price, + CAST(NULL AS NUMBER(38, 0)) AS previous_known_price, + 1299 AS first_price, + 1399 AS running_last_price, + 1399 AS partition_last_price, + 5 AS recency_rank, + 3 AS price_rank + UNION ALL + SELECT + 'LE-02' AS entry_id, + 1299 AS previous_price, + 1299 AS previous_known_price, + 1299 AS first_price, + 1399 AS running_last_price, + 1399 AS partition_last_price, + 4 AS recency_rank, + 1 AS price_rank + UNION ALL + SELECT + 'LE-03' AS entry_id, + CAST(NULL AS NUMBER(38, 0)) AS previous_price, + 1299 AS previous_known_price, + 1299 AS first_price, + 1399 AS running_last_price, + 1399 AS partition_last_price, + 3 AS recency_rank, + 4 AS price_rank + UNION ALL + SELECT + 'LE-04' AS entry_id, + 1199 AS previous_price, + 1199 AS previous_known_price, + 1299 AS first_price, + 1399 AS running_last_price, + 1399 AS partition_last_price, + 2 AS recency_rank, + 4 AS price_rank + UNION ALL + SELECT + 'LE-05' AS entry_id, + 1199 AS previous_price, + 1199 AS previous_known_price, + 1299 AS first_price, + 1399 AS running_last_price, + 1399 AS partition_last_price, + 1 AS recency_rank, + 2 AS price_rank + UNION ALL + SELECT + 'LE-06' AS entry_id, + CAST(NULL AS NUMBER(38, 0)) AS previous_price, + CAST(NULL AS NUMBER(38, 0)) AS previous_known_price, + 899 AS first_price, + 899 AS running_last_price, + 899 AS partition_last_price, + 1 AS recency_rank, + 1 AS price_rank +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_quantity_checks.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_quantity_checks.sql new file mode 100644 index 0000000000..0ada7d946f --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_quantity_checks.sql @@ -0,0 +1,59 @@ +TEST (name "int_edge_quantity_checks__casts_text_only_after_filtering_out_non_numbers"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + v.quantity_text + FROM VALUES + ('LE-01', 7, '007'), + ('LE-02', 7, '7.0'), + ('LE-03', 7, ' 7 '), + ('LE-04', 7, 'n/a'), + ('LE-05', 7, '1e0'), + ('LE-06', 7, '6.5'), + ('LE-07', 7, ''), + ('LE-08', 7, NULL) AS v(entry_id, quantity, quantity_text) +), + +__expected__int_edge_quantity_checks AS ( + SELECT + 'LE-01' AS entry_id, + '007' AS quantity_text, + FALSE AS text_equals_seven, + TRUE AS quantity_equals_seven_text, + 7 AS quantity_cast + UNION ALL + SELECT + 'LE-02' AS entry_id, + '7.0' AS quantity_text, + FALSE AS text_equals_seven, + TRUE AS quantity_equals_seven_text, + 7 AS quantity_cast + UNION ALL + SELECT + 'LE-03' AS entry_id, + ' 7 ' AS quantity_text, + FALSE AS text_equals_seven, + TRUE AS quantity_equals_seven_text, + 7 AS quantity_cast + UNION ALL + SELECT + 'LE-05' AS entry_id, + '1e0' AS quantity_text, + FALSE AS text_equals_seven, + TRUE AS quantity_equals_seven_text, + 1 AS quantity_cast + UNION ALL + SELECT + 'LE-06' AS entry_id, + '6.5' AS quantity_text, + FALSE AS text_equals_seven, + TRUE AS quantity_equals_seven_text, + 7 AS quantity_cast +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_channel_reasons.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_channel_reasons.sql new file mode 100644 index 0000000000..7ca7ac0239 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_channel_reasons.sql @@ -0,0 +1,151 @@ +TEST (name "rpt_edge_channel_reasons__aggregates_empty_and_all_missing_groups"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + v.channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + NULL AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + v.reason_code + FROM VALUES + ('LE-01', 'web', 2, 'REFUND'), + ('LE-02', 'web', 1, 'DAMAGED'), + ('LE-03', 'web', NULL, NULL), + ('LE-04', 'app', 1, NULL), + ('LE-05', 'app', 3, NULL), + ('LE-06', 'catalogue', 1, 'REFUND') AS v(entry_id, channel, quantity, reason_code) +), + +__expected__rpt_edge_channel_reasons AS ( + SELECT + 'app' AS channel, + 2 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + 4 AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'partner' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'phone' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'web' AS channel, + 3 AS lines, + 2 AS reasons, + 1 AS refunded_lines, + TRUE AS any_refund, + 3 AS total_quantity, + 'REFUND' AS max_reason, + 'REFUND,DAMAGED' AS reason_list, + PARSE_JSON('["REFUND","DAMAGED"]')::ARRAY AS reason_array, + PARSE_JSON('{"LE-01":"REFUND","LE-02":"DAMAGED"}')::OBJECT AS reasons_by_entry +) + +TEST (name "rpt_edge_channel_reasons__reports_every_channel_without_any_lines"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + CAST(NULL AS VARCHAR) AS entry_id, + CAST(NULL AS VARCHAR) AS order_ref, + CAST(NULL AS VARCHAR) AS channel, + CAST(NULL AS NUMBER(38, 0)) AS line_no, + CAST(NULL AS VARCHAR) AS sku, + CAST(NULL AS VARCHAR) AS warehouse_code, + CAST(NULL AS NUMBER(38, 0)) AS quantity, + CAST(NULL AS VARCHAR) AS quantity_text, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal, + CAST(NULL AS NUMBER(38, 0)) AS unit_price_minor, + CAST(NULL AS NUMBER(38, 10)) AS amount, + CAST(NULL AS VARCHAR) AS currency_code, + CAST(NULL AS FLOAT) AS fx_rate, + CAST(NULL AS VARCHAR) AS status, + CAST(NULL AS VARCHAR) AS reason_code + WHERE FALSE +), + +__expected__rpt_edge_channel_reasons AS ( + SELECT + 'app' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'partner' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'phone' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry + UNION ALL + SELECT + 'web' AS channel, + 0 AS lines, + 0 AS reasons, + 0 AS refunded_lines, + CAST(NULL AS BOOLEAN) AS any_refund, + CAST(NULL AS NUMBER(38, 0)) AS total_quantity, + CAST(NULL AS VARCHAR) AS max_reason, + '' AS reason_list, + PARSE_JSON('[]')::ARRAY AS reason_array, + PARSE_JSON('{}')::OBJECT AS reasons_by_entry +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_stock_reconciliation.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_stock_reconciliation.sql new file mode 100644 index 0000000000..f7cdace985 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_stock_reconciliation.sql @@ -0,0 +1,65 @@ +TEST (name "rpt_edge_stock_reconciliation__keeps_rows_found_on_only_one_side"); + +WITH __ref__stg_edge_ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + v.sku, + v.warehouse_code, + v.quantity::NUMBER(38, 0) AS quantity, + NULL AS quantity_text, + NULL AS quantity_from_text, + NULL AS quantity_text_decimal, + NULL AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + v.status + FROM VALUES + ('LE-01', 'MALLARD-01', 'POND', 3, 'allocated'), + ('LE-02', 'MALLARD-01', 'POND', 2, 'COUNTED'), + ('LE-03', 'TEAL-02', 'POND', 4, 'Allocated'), + ('LE-04', 'TEAL-02', 'NEST', 1, 'counted'), + ('LE-05', 'EIDER-03', NULL, 5, 'allocated'), + ('LE-06', 'EIDER-03', NULL, 5, 'counted'), + ('LE-07', 'PINTAIL-04', 'MARSH', 9, 'pending') AS v(entry_id, sku, warehouse_code, quantity, status) +), + +__expected__rpt_edge_stock_reconciliation AS ( + SELECT + 'EIDER-03' AS sku, + CAST(NULL AS VARCHAR) AS warehouse_code, + 5 AS allocated_units, + CAST(NULL AS NUMBER(38, 0)) AS counted_units, + -5 AS variance_units + UNION ALL + SELECT + 'EIDER-03' AS sku, + CAST(NULL AS VARCHAR) AS warehouse_code, + CAST(NULL AS NUMBER(38, 0)) AS allocated_units, + 5 AS counted_units, + 5 AS variance_units + UNION ALL + SELECT + 'MALLARD-01' AS sku, + 'POND' AS warehouse_code, + 3 AS allocated_units, + 2 AS counted_units, + -1 AS variance_units + UNION ALL + SELECT + 'TEAL-02' AS sku, + 'NEST' AS warehouse_code, + CAST(NULL AS NUMBER(38, 0)) AS allocated_units, + 1 AS counted_units, + 1 AS variance_units + UNION ALL + SELECT + 'TEAL-02' AS sku, + 'POND' AS warehouse_code, + 4 AS allocated_units, + CAST(NULL AS NUMBER(38, 0)) AS counted_units, + -4 AS variance_units +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_customer_profiles.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_customer_profiles.sql new file mode 100644 index 0000000000..d1d960deca --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_customer_profiles.sql @@ -0,0 +1,254 @@ +TEST (name "stg_edge_customer_profiles__ranks_a_snapshot_without_update_time_first"); + +WITH __source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + NULL AS full_name, + NULL AS previous_full_name, + NULL AS first_name, + NULL AS middle_name, + NULL AS last_name, + NULL AS address_line1, + NULL AS address_line2, + v.loyalty_tier, + NULL AS partner_order_number, + v.profile_updated_at::TIMESTAMP_NTZ AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', 'Duckling', '2026-03-01 09:00:00'), + ('PR-2', 'C-1', 'Golden Duck', NULL), + ('PR-3', 'C-2', 'Duckling', '2026-03-01 09:00:00'), + ('PR-4', 'C-2', 'Mallard', '2026-03-02 09:00:00'), + ('PR-5', 'C-3', 'Mallard', '2026-03-02 09:00:00'), + ('PR-6', 'C-3', 'Teal', '2026-03-02 09:00:00'), + ('PR-7', NULL, 'BLOCKED', NULL) AS v(profile_id, customer_id, loyalty_tier, profile_updated_at) +), + +__expected__stg_edge_customer_profiles AS ( + SELECT + 'C-1' AS customer_id, + 'PR-2' AS profile_id, + 'Golden Duck' AS loyalty_tier + UNION ALL + SELECT + 'C-2' AS customer_id, + 'PR-4' AS profile_id, + 'Mallard' AS loyalty_tier + UNION ALL + SELECT + 'C-3' AS customer_id, + 'PR-6' AS profile_id, + 'Teal' AS loyalty_tier +) + +TEST (name "stg_edge_customer_profiles__joins_name_and_address_parts_that_may_be_missing"); + +WITH __source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + NULL AS full_name, + NULL AS previous_full_name, + v.first_name, + v.middle_name, + v.last_name, + v.address_line1, + v.address_line2, + NULL AS loyalty_tier, + NULL AS partner_order_number, + NULL AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', 'Ada', 'Mallard', 'Drake', '1 Pond Lane', 'Flat 2'), + ('PR-2', 'C-2', 'Ada', NULL, 'Drake', '1 Pond Lane', NULL), + ('PR-3', 'C-3', 'Ada', '', 'Drake', '1 Pond Lane', ''), + ('PR-4', 'C-4', NULL, NULL, 'Drake', NULL, 'Flat 2'), + ('PR-5', 'C-5', 'Ada', NULL, NULL, NULL, NULL) AS v(profile_id, customer_id, first_name, middle_name, last_name, address_line1, address_line2) +), + +__expected__stg_edge_customer_profiles AS ( + SELECT + 'C-1' AS customer_id, + 'Ada Mallard Drake' AS display_name, + 'Ada Drake' AS display_name_concat, + 'Ada Mallard Drake' AS display_name_skip_nulls, + '1 Pond Lane, Flat 2' AS postal_address + UNION ALL + SELECT + 'C-2' AS customer_id, + CAST(NULL AS VARCHAR) AS display_name, + 'Ada Drake' AS display_name_concat, + CAST(NULL AS VARCHAR) AS display_name_skip_nulls, + CAST(NULL AS VARCHAR) AS postal_address + UNION ALL + SELECT + 'C-3' AS customer_id, + 'Ada Drake' AS display_name, + 'Ada Drake' AS display_name_concat, + 'Ada Drake' AS display_name_skip_nulls, + '1 Pond Lane, ' AS postal_address + UNION ALL + SELECT + 'C-4' AS customer_id, + CAST(NULL AS VARCHAR) AS display_name, + CAST(NULL AS VARCHAR) AS display_name_concat, + CAST(NULL AS VARCHAR) AS display_name_skip_nulls, + CAST(NULL AS VARCHAR) AS postal_address + UNION ALL + SELECT + 'C-5' AS customer_id, + CAST(NULL AS VARCHAR) AS display_name, + CAST(NULL AS VARCHAR) AS display_name_concat, + CAST(NULL AS VARCHAR) AS display_name_skip_nulls, + CAST(NULL AS VARCHAR) AS postal_address +) + +TEST (name "stg_edge_customer_profiles__measures_and_recases_names_in_many_scripts"); + +WITH __source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + v.full_name, + NULL AS previous_full_name, + v.first_name, + NULL AS middle_name, + NULL AS last_name, + NULL AS address_line1, + NULL AS address_line2, + NULL AS loyalty_tier, + NULL AS partner_order_number, + NULL AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', 'größe ente', 'jürgen'), + ('PR-2', 'C-2', 'e\u0301lodie o''neill-smith', 'e\u0301lodie'), + ('PR-3', 'C-3', '\u0133sselmeer van der eend', '\u0133sbrand'), + ('PR-4', 'C-4', '\u10DC\u10D8\u10DC\u10DD \u10D1\u10D4\u10E0\u10D8\u10EB\u10D4', 'ab'), + ('PR-5', 'C-5', 'ann (dup)', 'ann'), + ('PR-6', 'C-6', 'mallard 🦆 fan', '🦆'), + ('PR-7', 'C-7', 'STRASSE straße', 'straße') AS v(profile_id, customer_id, full_name, first_name) +), + +__expected__stg_edge_customer_profiles AS ( + SELECT + 'C-1' AS customer_id, + 'Größe Ente' AS name_title_case, + 'GRÖSSE ENTE' AS name_upper, + 10 AS name_length, + 'g' AS name_initial, + '**jürgen' AS first_name_padded + UNION ALL + SELECT + 'C-2' AS customer_id, + 'E\u0301lodie O''neill-Smith' AS name_title_case, + 'E\u0301LODIE O''NEILL-SMITH' AS name_upper, + 21 AS name_length, + 'e' AS name_initial, + '*e\u0301lodie' AS first_name_padded + UNION ALL + SELECT + 'C-3' AS customer_id, + 'IJsselmeer Van Der Eend' AS name_title_case, + 'IJSSELMEER VAN DER EEND' AS name_upper, + 22 AS name_length, + 'ij' AS name_initial, + '*ijsbrand' AS first_name_padded + UNION ALL + SELECT + 'C-4' AS customer_id, + 'ნინო ბერიძე' AS name_title_case, + 'ნინო ბერიძე' AS name_upper, + 11 AS name_length, + 'ნ' AS name_initial, + '******ab' AS first_name_padded + UNION ALL + SELECT + 'C-5' AS customer_id, + 'Ann (Dup)' AS name_title_case, + 'ANN (DUP)' AS name_upper, + 9 AS name_length, + 'a' AS name_initial, + '*****ann' AS first_name_padded + UNION ALL + SELECT + 'C-6' AS customer_id, + 'Mallard 🦆 Fan' AS name_title_case, + 'MALLARD 🦆 FAN' AS name_upper, + 13 AS name_length, + 'm' AS name_initial, + '*******🦆' AS first_name_padded + UNION ALL + SELECT + 'C-7' AS customer_id, + 'Strasse Straße' AS name_title_case, + 'STRASSE STRASSE' AS name_upper, + 14 AS name_length, + 'S' AS name_initial, + '**straße' AS first_name_padded +) + +TEST (name "stg_edge_customer_profiles__compares_order_numbers_past_double_precision_with_floats"); + +WITH __source__edge__customer_profiles AS ( + SELECT + v.profile_id, + v.customer_id, + NULL AS full_name, + NULL AS previous_full_name, + NULL AS first_name, + NULL AS middle_name, + NULL AS last_name, + NULL AS address_line1, + NULL AS address_line2, + NULL AS loyalty_tier, + v.partner_order_number::NUMBER(38, 0) AS partner_order_number, + NULL AS profile_updated_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('PR-1', 'C-1', '9007199254740992'), + ('PR-2', 'C-2', '9007199254740993'), + ('PR-3', 'C-3', '9007199254740991'), + ('PR-4', 'C-4', '12345678901234567890'), + ('PR-5', 'C-5', '10000000000000000001'), + ('PR-6', 'C-6', '-9223372036854775809') AS v(profile_id, customer_id, partner_order_number) +), + +__expected__stg_edge_customer_profiles AS ( + SELECT + 'C-1' AS customer_id, + 9007199254740992 AS partner_order_number, + '9007199254740992.0'::FLOAT AS partner_order_number_float, + TRUE AS is_long_order_number + UNION ALL + SELECT + 'C-2' AS customer_id, + 9007199254740993 AS partner_order_number, + '9007199254740992.0'::FLOAT AS partner_order_number_float, + TRUE AS is_long_order_number + UNION ALL + SELECT + 'C-3' AS customer_id, + 9007199254740991 AS partner_order_number, + '9007199254740991.0'::FLOAT AS partner_order_number_float, + FALSE AS is_long_order_number + UNION ALL + SELECT + 'C-4' AS customer_id, + 12345678901234567890 AS partner_order_number, + '1.2345678901234567e+19'::FLOAT AS partner_order_number_float, + FALSE AS is_long_order_number + UNION ALL + SELECT + 'C-5' AS customer_id, + 10000000000000000001 AS partner_order_number, + '1e+19'::FLOAT AS partner_order_number_float, + TRUE AS is_long_order_number + UNION ALL + SELECT + 'C-6' AS customer_id, + -9223372036854775809 AS partner_order_number, + '-9.223372036854776e+18'::FLOAT AS partner_order_number_float, + FALSE AS is_long_order_number +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ledger_entries.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ledger_entries.sql new file mode 100644 index 0000000000..d2d144d22a --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ledger_entries.sql @@ -0,0 +1,172 @@ +TEST (name "stg_edge_ledger_entries__reads_quantity_text_in_every_number_spelling"); + +WITH __source__edge__ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + NULL AS quantity, + v.quantity_text, + NULL AS unit_price_minor, + NULL AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + NULL AS reason_code, + NULL AS customer_tier, + NULL AS changed_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('LE-01', '007'), + ('LE-02', '+7'), + ('LE-03', '1e0'), + ('LE-04', ' 1.5 '), + ('LE-05', '-0'), + ('LE-06', '1,000'), + ('LE-07', 'n/a'), + ('LE-08', ''), + ('LE-09', '1.005'), + ('LE-10', '2.5'), + ('LE-11', '\u0661\u0662'), + ('LE-12', '0x1F'), + ('LE-13', '1_000'), + ('LE-14', '.5'), + ('LE-15', '5.'), + ('LE-16', '1234567890123456.789') AS v(entry_id, quantity_text) +), + +__expected__stg_edge_ledger_entries AS ( + SELECT + 'LE-01' AS entry_id, + 7.00::NUMBER(18, 2) AS quantity_from_text, + 7 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-02' AS entry_id, + 7.00::NUMBER(18, 2) AS quantity_from_text, + 7 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-03' AS entry_id, + 1.00::NUMBER(18, 2) AS quantity_from_text, + 1 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-04' AS entry_id, + 1.50::NUMBER(18, 2) AS quantity_from_text, + 2 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-05' AS entry_id, + 0.00::NUMBER(18, 2) AS quantity_from_text, + 0 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-06' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-07' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-08' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-09' AS entry_id, + 1.01::NUMBER(18, 2) AS quantity_from_text, + 1 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-10' AS entry_id, + 2.50::NUMBER(18, 2) AS quantity_from_text, + 3 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-11' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-12' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-13' AS entry_id, + CAST(NULL AS NUMBER(18, 2)) AS quantity_from_text, + CAST(NULL AS NUMBER(18, 0)) AS quantity_text_decimal + UNION ALL + SELECT + 'LE-14' AS entry_id, + 0.50::NUMBER(18, 2) AS quantity_from_text, + 1 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-15' AS entry_id, + 5.00::NUMBER(18, 2) AS quantity_from_text, + 5 AS quantity_text_decimal + UNION ALL + SELECT + 'LE-16' AS entry_id, + 1234567890123456.79::NUMBER(18, 2) AS quantity_from_text, + 1234567890123457 AS quantity_text_decimal +) + +TEST (name "stg_edge_ledger_entries__keeps_38_digit_amounts_exact"); + +WITH __source__edge__ledger_entries AS ( + SELECT + v.entry_id, + NULL AS order_ref, + NULL AS channel, + NULL AS line_no, + NULL AS sku, + NULL AS warehouse_code, + NULL AS quantity, + NULL AS quantity_text, + NULL AS unit_price_minor, + v.amount::NUMBER(38, 10) AS amount, + NULL AS currency_code, + NULL AS fx_rate, + NULL AS status, + NULL AS reason_code, + NULL AS customer_tier, + NULL AS changed_at, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('LE-01', '9999999999999999999999999999.9999999999'), + ('LE-02', '1234567890123456789012345678.0123456789'), + ('LE-03', '-9999999999999999999999999999.9999999999'), + ('LE-04', '0.0000000001'), + ('LE-05', '9007199254740993.0000000001') AS v(entry_id, amount) +), + +__expected__stg_edge_ledger_entries AS ( + SELECT + 'LE-01' AS entry_id, + 9999999999999999999999999999.9999999999::NUMBER(38, 10) AS amount + UNION ALL + SELECT + 'LE-02' AS entry_id, + 1234567890123456789012345678.0123456789::NUMBER(38, 10) AS amount + UNION ALL + SELECT + 'LE-03' AS entry_id, + -9999999999999999999999999999.9999999999::NUMBER(38, 10) AS amount + UNION ALL + SELECT + 'LE-04' AS entry_id, + 0.0000000001::NUMBER(38, 10) AS amount + UNION ALL + SELECT + 'LE-05' AS entry_id, + 9007199254740993.0000000001::NUMBER(38, 10) AS amount +) diff --git a/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ticket_notes.sql b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ticket_notes.sql new file mode 100644 index 0000000000..73143e52ab --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/staging/edge_cases/test_stg_edge_ticket_notes.sql @@ -0,0 +1,257 @@ +TEST (name "stg_edge_ticket_notes__finds_order_references_with_anchored_and_unanchored_patterns"); + +WITH __source__edge__ticket_notes AS ( + SELECT + v.note_id, + v.body, + v.tags_text, + NULL AS utm_source, + NULL AS discount_code, + NULL AS sku, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('TN-01', 'Customer O''Brien asks about DS-O-1001 and DS-O-1002; refund\tplease', 'bath#gift#teal'), + ('TN-02', 'Order ds-o-77 was REFUNDED.\nPath C:\\ducks\\pond # not a comment', '#leading'), + ('TN-03', 'DS-O-3003', 'single'), + ('TN-04', 'No order here, refunds are slow', ''), + ('TN-05', NULL, NULL), + ('TN-06', 'Refunded DS-O-12x and DS-O-0042', 'a##b') AS v(note_id, body, tags_text) +), + +__expected__stg_edge_ticket_notes AS ( + SELECT + 'TN-01' AS note_id, + 'DS-O-1001' AS first_order_ref, + 'DS-O-1001' AS first_order_ref_dollar, + '1002' AS second_order_number, + FALSE AS mentions_order_rlike, + TRUE AS mentions_order_regexp_like, + 2 AS order_ref_count, + 'refund' AS refund_word, + 'Customer O''Brien asks about DS-O-1001 and DS-O-1002; refund please' AS body_collapsed, + TRUE AS quotes_customer, + 'gift' AS second_hash_tag + UNION ALL + SELECT + 'TN-02' AS note_id, + CAST(NULL AS VARCHAR) AS first_order_ref, + CAST(NULL AS VARCHAR) AS first_order_ref_dollar, + CAST(NULL AS VARCHAR) AS second_order_number, + FALSE AS mentions_order_rlike, + TRUE AS mentions_order_regexp_like, + 0 AS order_ref_count, + 'REFUNDED' AS refund_word, + 'Order ds-o-77 was REFUNDED. Path C:\\ducks\\pond # not a comment' AS body_collapsed, + FALSE AS quotes_customer, + 'leading' AS second_hash_tag + UNION ALL + SELECT + 'TN-03' AS note_id, + 'DS-O-3003' AS first_order_ref, + 'DS-O-3003' AS first_order_ref_dollar, + CAST(NULL AS VARCHAR) AS second_order_number, + TRUE AS mentions_order_rlike, + TRUE AS mentions_order_regexp_like, + 1 AS order_ref_count, + CAST(NULL AS VARCHAR) AS refund_word, + 'DS-O-3003' AS body_collapsed, + FALSE AS quotes_customer, + '' AS second_hash_tag + UNION ALL + SELECT + 'TN-04' AS note_id, + CAST(NULL AS VARCHAR) AS first_order_ref, + CAST(NULL AS VARCHAR) AS first_order_ref_dollar, + CAST(NULL AS VARCHAR) AS second_order_number, + FALSE AS mentions_order_rlike, + FALSE AS mentions_order_regexp_like, + 0 AS order_ref_count, + 'refund' AS refund_word, + 'No order here, refunds are slow' AS body_collapsed, + FALSE AS quotes_customer, + '' AS second_hash_tag + UNION ALL + SELECT + 'TN-05' AS note_id, + CAST(NULL AS VARCHAR) AS first_order_ref, + CAST(NULL AS VARCHAR) AS first_order_ref_dollar, + CAST(NULL AS VARCHAR) AS second_order_number, + CAST(NULL AS BOOLEAN) AS mentions_order_rlike, + CAST(NULL AS BOOLEAN) AS mentions_order_regexp_like, + CAST(NULL AS NUMBER(38, 0)) AS order_ref_count, + CAST(NULL AS VARCHAR) AS refund_word, + CAST(NULL AS VARCHAR) AS body_collapsed, + CAST(NULL AS BOOLEAN) AS quotes_customer, + CAST(NULL AS VARCHAR) AS second_hash_tag + UNION ALL + SELECT + 'TN-06' AS note_id, + 'DS-O-12' AS first_order_ref, + 'DS-O-12' AS first_order_ref_dollar, + '0042' AS second_order_number, + FALSE AS mentions_order_rlike, + TRUE AS mentions_order_regexp_like, + 2 AS order_ref_count, + 'Refunded' AS refund_word, + 'Refunded DS-O-12x and DS-O-0042' AS body_collapsed, + FALSE AS quotes_customer, + '' AS second_hash_tag +) + +TEST (name "stg_edge_ticket_notes__matches_codes_by_pattern_lists_and_escapes"); + +WITH __source__edge__ticket_notes AS ( + SELECT + v.note_id, + NULL AS body, + NULL AS tags_text, + v.utm_source, + v.discount_code, + NULL AS sku, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('TN-01', 'newsletter', '50%OFF'), + ('TN-02', 'Email', 'STAFF10'), + ('TN-03', 'partner_feed', '50PERCENT'), + ('TN-04', 'NEWS', 'test5'), + ('TN-05', NULL, NULL), + ('TN-06', 'gmail', '50%'), + ('TN-07', 'news%', 'TEST5') AS v(note_id, utm_source, discount_code) +), + +__expected__stg_edge_ticket_notes AS ( + SELECT + 'TN-01' AS note_id, + TRUE AS is_newsletter_visit, + TRUE AS is_customer_code, + TRUE AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-02' AS note_id, + TRUE AS is_newsletter_visit, + FALSE AS is_customer_code, + FALSE AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-03' AS note_id, + FALSE AS is_newsletter_visit, + TRUE AS is_customer_code, + FALSE AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-04' AS note_id, + TRUE AS is_newsletter_visit, + TRUE AS is_customer_code, + FALSE AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-05' AS note_id, + CAST(NULL AS BOOLEAN) AS is_newsletter_visit, + CAST(NULL AS BOOLEAN) AS is_customer_code, + CAST(NULL AS BOOLEAN) AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-06' AS note_id, + TRUE AS is_newsletter_visit, + TRUE AS is_customer_code, + TRUE AS is_fifty_percent_code + UNION ALL + SELECT + 'TN-07' AS note_id, + TRUE AS is_newsletter_visit, + FALSE AS is_customer_code, + FALSE AS is_fifty_percent_code +) + +TEST (name "stg_edge_ticket_notes__splits_skus_at_edge_positions"); + +WITH __source__edge__ticket_notes AS ( + SELECT + v.note_id, + NULL AS body, + NULL AS tags_text, + NULL AS utm_source, + NULL AS discount_code, + v.sku, + '2026-03-05 00:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('TN-01', 'MALLARD-01'), + ('TN-02', 'DUCK--'), + ('TN-03', ''), + ('TN-04', 'A-B-C-D'), + ('TN-05', NULL), + ('TN-06', 'EIDER'), + ('TN-07', '-LEADING') AS v(note_id, sku) +), + +__expected__stg_edge_ticket_notes AS ( + SELECT + 'TN-01' AS note_id, + 'MALLARD' AS sku_part_zero, + '01' AS sku_part_last, + '' AS sku_part_beyond, + 'MALLARD-01' AS sku_part_no_delimiter, + 'MAL' AS sku_from_zero, + '01' AS sku_last_two, + 'MAL' AS sku_before_start + UNION ALL + SELECT + 'TN-02' AS note_id, + 'DUCK' AS sku_part_zero, + '' AS sku_part_last, + '' AS sku_part_beyond, + 'DUCK--' AS sku_part_no_delimiter, + 'DUC' AS sku_from_zero, + '--' AS sku_last_two, + '' AS sku_before_start + UNION ALL + SELECT + 'TN-03' AS note_id, + '' AS sku_part_zero, + '' AS sku_part_last, + '' AS sku_part_beyond, + '' AS sku_part_no_delimiter, + '' AS sku_from_zero, + '' AS sku_last_two, + '' AS sku_before_start + UNION ALL + SELECT + 'TN-04' AS note_id, + 'A' AS sku_part_zero, + 'D' AS sku_part_last, + '' AS sku_part_beyond, + 'A-B-C-D' AS sku_part_no_delimiter, + 'A-B' AS sku_from_zero, + '-D' AS sku_last_two, + '' AS sku_before_start + UNION ALL + SELECT + 'TN-05' AS note_id, + CAST(NULL AS VARCHAR) AS sku_part_zero, + CAST(NULL AS VARCHAR) AS sku_part_last, + CAST(NULL AS VARCHAR) AS sku_part_beyond, + CAST(NULL AS VARCHAR) AS sku_part_no_delimiter, + CAST(NULL AS VARCHAR) AS sku_from_zero, + CAST(NULL AS VARCHAR) AS sku_last_two, + CAST(NULL AS VARCHAR) AS sku_before_start + UNION ALL + SELECT + 'TN-06' AS note_id, + 'EIDER' AS sku_part_zero, + 'EIDER' AS sku_part_last, + '' AS sku_part_beyond, + 'EIDER' AS sku_part_no_delimiter, + 'EID' AS sku_from_zero, + 'ER' AS sku_last_two, + '' AS sku_before_start + UNION ALL + SELECT + 'TN-07' AS note_id, + '' AS sku_part_zero, + 'LEADING' AS sku_part_last, + '' AS sku_part_beyond, + '-LEADING' AS sku_part_no_delimiter, + '-LE' AS sku_from_zero, + 'NG' AS sku_last_two, + '' AS sku_before_start +) From 9bced22c9a959e057a5534b513e7794b6aaece09 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:54:44 +0000 Subject: [PATCH 4/6] test(benchmarks): add DuckShop edge scenarios and incremental lifecycle cases Add partner listings and a shipment delivery change log to the edge-case area, with scenarios and Snowflake-captured unit tests for: event documents built from typed values keeping their types across model boundaries; mixed-type arrays and VARIANT expected output compared with expected_columns "all"; a star published under a lower-case contract and an export with quoted camelCase columns, read from a source whose columns are quoted camelCase; and a cursor-window incremental model built in a scenario over fixture loads a millennium apart. The delivery change log also replays as two loads (the edge_delivery_batch variable), and a singular audit checks that the merged incremental status equals a full rebuild after late, redelivered and deleted versions. A disabled cursor delete_insert model and its unit test record the out-of-window delta case as an acceptance test for a future guard. --- .../edge_delivery_status_matches_rebuild.sql | 95 +++++++++++ .../edge_cases/int_edge_delivery_status.sql | 83 ++++++++++ .../int_edge_delivery_status_rebuilt.sql | 53 ++++++ .../int_edge_delivery_update_versions.sql | 52 ++++++ .../edge_cases/dim_edge_partner_listings.sql | 21 +++ .../edge_cases/fct_edge_delivery_events.sql | 49 ++++++ .../rpt_edge_partner_listing_export.sql | 30 ++++ .../rpt_edge_payload_price_types.sql | 42 +++++ .../edge_cases/stg_edge_partner_listings.sql | 28 ++++ .../project/python/loaders/_edge_cases.py | 61 +++++++ .../project/python/loaders/edge_cases.py | 16 +- .../duckshop/project/sources/edge_cases.yml | 61 +++++++ .../duckshop/project/sqlbuild_project.toml | 21 +++ .../scenario.json | 68 ++++++++ .../sources/edge__delivery_updates.jsonl | 5 + .../refs/stg_edge_partner_listings.jsonl | 3 + .../scenario.json | 38 +++++ .../scenario.json | 58 +++++++ .../sources/edge__partner_payloads.jsonl | 2 + .../scenario.json | 58 +++++++ .../sources/edge__partner_payloads.jsonl | 3 + ...sor_model_over_far_apart_fixture_loads.sql | 61 +++++++ ...tings__star_contract_and_quoted_export.sql | 40 +++++ ...es__mixed_type_arrays_flatten_to_lines.sql | 69 ++++++++ ...t_chain__typed_payload_survives_models.sql | 45 ++++++ .../test_int_edge_delivery_status.sql | 151 ++++++++++++++++++ .../test_int_edge_delivery_status_rebuilt.sql | 82 ++++++++++ ...test_int_edge_delivery_update_versions.sql | 116 ++++++++++++++ .../test_int_edge_order_line_labels.sql | 13 +- .../test_dim_edge_partner_listings.sql | 23 +++ .../test_fct_edge_delivery_events.sql | 35 ++++ .../test_rpt_edge_partner_listing_export.sql | 23 +++ .../test_rpt_edge_payload_price_types.sql | 56 +++++++ 33 files changed, 1554 insertions(+), 7 deletions(-) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/_sqlbuild/audits/singular/edge_delivery_status_matches_rebuild.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_rebuilt.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_update_versions.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/dim_edge_partner_listings.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_delivery_events.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_payload_price_types.sql create mode 100644 benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_listings.sql create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/scenario.json create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/sources/edge__delivery_updates.jsonl create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/refs/stg_edge_partner_listings.jsonl create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/scenario.json create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/scenario.json create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/sources/edge__partner_payloads.jsonl create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/scenario.json create mode 100644 benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/sources/edge__partner_payloads.jsonl create mode 100644 benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_delivery_status__cursor_model_over_far_apart_fixture_loads.sql create mode 100644 benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_partner_listings__star_contract_and_quoted_export.sql create mode 100644 benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_payload_lines__mixed_type_arrays_flatten_to_lines.sql create mode 100644 benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_variant_chain__typed_payload_survives_models.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_rebuilt.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_update_versions.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_dim_edge_partner_listings.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_delivery_events.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_payload_price_types.sql diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/_sqlbuild/audits/singular/edge_delivery_status_matches_rebuild.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/_sqlbuild/audits/singular/edge_delivery_status_matches_rebuild.sql new file mode 100644 index 0000000000..d0daeddcb4 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/_sqlbuild/audits/singular/edge_delivery_status_matches_rebuild.sql @@ -0,0 +1,95 @@ +AUDIT ( + name "edge_delivery_status_matches_rebuild", + severity error +); + +-- The incremental delivery status must equal a rebuild from the whole change log after every +-- build, whatever order the versions arrived in. Rows are reported from both sides. +WITH incremental AS ( + SELECT + s.shipment_id, + s.order_ref, + s.delivery_status, + s.ordered_on, + s.delivered_on, + s.is_deleted, + s.latest_seq, + s.source_loaded_at + FROM __ref("int_edge_delivery_status") AS s +), + +rebuilt AS ( + SELECT + r.shipment_id, + r.order_ref, + r.delivery_status, + r.ordered_on, + r.delivered_on, + r.is_deleted, + r.latest_seq, + r.source_loaded_at + FROM __ref("int_edge_delivery_status_rebuilt") AS r +), + +only_incremental AS ( + SELECT + i.shipment_id, + i.order_ref, + i.delivery_status, + i.ordered_on, + i.delivered_on, + i.is_deleted, + i.latest_seq, + i.source_loaded_at + FROM incremental AS i + EXCEPT + SELECT + r.shipment_id, + r.order_ref, + r.delivery_status, + r.ordered_on, + r.delivered_on, + r.is_deleted, + r.latest_seq, + r.source_loaded_at + FROM rebuilt AS r +), + +only_rebuilt AS ( + SELECT + r.shipment_id, + r.order_ref, + r.delivery_status, + r.ordered_on, + r.delivered_on, + r.is_deleted, + r.latest_seq, + r.source_loaded_at + FROM rebuilt AS r + EXCEPT + SELECT + i.shipment_id, + i.order_ref, + i.delivery_status, + i.ordered_on, + i.delivered_on, + i.is_deleted, + i.latest_seq, + i.source_loaded_at + FROM incremental AS i +), + +final AS ( + SELECT + 'incremental' AS side, + o.shipment_id + FROM only_incremental AS o + UNION ALL + SELECT + 'rebuilt' AS side, + b.shipment_id + FROM only_rebuilt AS b +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status.sql new file mode 100644 index 0000000000..1e689c401b --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status.sql @@ -0,0 +1,83 @@ +MODEL ( + description """ + One row per shipment with the state of its latest delivery version, deletes kept as flagged + rows. Merged incrementally: every shipment with a version loaded in the window is recomputed + from its whole history, read from a model that is not a cursor input and so is not cut to the + window, which gives late older versions, redeliveries and deletes the same rows as a rebuild. + """, + tags [edge_case], + materialized incremental, + incremental_strategy merge, + unique_key [shipment_id], + cursor source_loaded_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + edge__delivery_updates _loaded_at, + ), + replay_on_change full, + columns ( + shipment_id (type VARCHAR, nullable false, audits [not_null, unique]), + order_ref (type VARCHAR), + delivery_status (type VARCHAR), + ordered_on (type DATE), + delivered_on (type DATE), + is_deleted (type BOOLEAN), + latest_seq (type NUMBER(38, 0)), + source_loaded_at (type TIMESTAMP_NTZ, nullable false, audits [not_null]), + ), +); + +WITH window_updates AS ( + SELECT u.shipment_id + FROM __source("edge__delivery_updates") AS u + WHERE + u._loaded_at >= __cursor_start() + AND u._loaded_at < __cursor_end() +), + +touched AS ( + SELECT DISTINCT w.shipment_id + FROM window_updates AS w +), + +history AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on, + v.delivered_on, + v.is_delete, + v.version_seq, + MAX(v.loaded_at) OVER (PARTITION BY v.shipment_id) AS source_loaded_at + FROM __ref("int_edge_delivery_update_versions") AS v + WHERE + v.loaded_at < __cursor_end() + AND v.shipment_id IN ( + SELECT t.shipment_id + FROM touched AS t + ) +), + +final AS ( + SELECT + h.shipment_id, + h.order_ref, + h.delivery_status, + h.ordered_on, + h.delivered_on, + h.is_delete AS is_deleted, + h.version_seq AS latest_seq, + h.source_loaded_at + FROM history AS h + QUALIFY + ROW_NUMBER() OVER ( + PARTITION BY h.shipment_id + ORDER BY h.version_seq DESC, h.delivery_id DESC + ) = 1 +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_rebuilt.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_rebuilt.sql new file mode 100644 index 0000000000..6dc96f8103 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_rebuilt.sql @@ -0,0 +1,53 @@ +MODEL ( + description """ + One row per shipment with the state of its latest delivery version over the whole change log, + deletes kept as flagged rows: the full-history reference the incremental delivery status must + equal after any sequence of loads + """, + tags [edge_case], + materialized view, + columns ( + shipment_id (type VARCHAR, nullable false, audits [not_null, unique]), + order_ref (type VARCHAR), + delivery_status (type VARCHAR), + ordered_on (type DATE), + delivered_on (type DATE), + is_deleted (type BOOLEAN), + latest_seq (type NUMBER(38, 0)), + source_loaded_at (type TIMESTAMP_NTZ), + ), +); + +WITH updates AS ( + SELECT + u.delivery_id, + u.shipment_id, + u.order_ref, + u.delivery_status, + u.ordered_on, + u.delivered_on, + u._op, + u._seq, + MAX(u._loaded_at) OVER (PARTITION BY u.shipment_id) AS source_loaded_at + FROM __source("edge__delivery_updates") AS u +), + +final AS ( + SELECT + u.shipment_id, + u.order_ref, + u.delivery_status, + u.ordered_on, + u.delivered_on, + ( + u._op = 'D' + ) AS is_deleted, + u._seq AS latest_seq, + u.source_loaded_at + FROM updates AS u + QUALIFY + ROW_NUMBER() OVER (PARTITION BY u.shipment_id ORDER BY u._seq DESC, u.delivery_id DESC) = 1 +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_update_versions.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_update_versions.sql new file mode 100644 index 0000000000..d813aca805 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_update_versions.sql @@ -0,0 +1,52 @@ +MODEL ( + description """ + One row per delivered version of a shipment's delivery change log, every load included, for + models that recompute a shipment from its whole history + """, + tags [edge_case], + materialized view, + columns ( + delivery_id (type VARCHAR, nullable false, audits [not_null, unique]), + shipment_id (type VARCHAR, nullable false, audits [not_null]), + order_ref (type VARCHAR), + delivery_status (type VARCHAR), + ordered_on (type DATE), + delivered_on (type DATE), + is_delete (type BOOLEAN), + version_seq (type NUMBER(38, 0)), + loaded_at (type TIMESTAMP_NTZ), + ), +); + +WITH updates AS ( + SELECT + u.delivery_id, + u.shipment_id, + u.order_ref, + u.delivery_status, + u.ordered_on, + u.delivered_on, + u._op, + u._seq, + u._loaded_at + FROM __source("edge__delivery_updates") AS u +), + +final AS ( + SELECT + u.delivery_id, + u.shipment_id, + u.order_ref, + u.delivery_status, + u.ordered_on, + u.delivered_on, + ( + u._op = 'D' + ) AS is_delete, + u._seq AS version_seq, + u._loaded_at AS loaded_at + FROM updates AS u +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/dim_edge_partner_listings.sql b/benchmarks/duckshop/project/models/marts/edge_cases/dim_edge_partner_listings.sql new file mode 100644 index 0000000000..062ed758f5 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/dim_edge_partner_listings.sql @@ -0,0 +1,21 @@ +MODEL ( + description """ + One row per partner listing, published under an enforced contract through a star over the + staging model, whose columns Snowflake names in upper case + """, + tags [edge_case], + contract enforced, + columns ( + listing_id (type VARCHAR, nullable false, audits [not_null, unique]), + unit_price_minor (type NUMBER(38, 0)), + listing_status (type VARCHAR), + ), +); + +WITH final AS ( + SELECT s.* + FROM __ref("stg_edge_partner_listings") AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_delivery_events.sql b/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_delivery_events.sql new file mode 100644 index 0000000000..c0bc6ae7dd --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/fct_edge_delivery_events.sql @@ -0,0 +1,49 @@ +MODEL ( + description """ + One row per delivered shipment version, rebuilt by delivery date while its input is filtered by + order date. Disabled: a rerun of a window inserts deliveries that fall outside the deleted + window again, so they duplicate. It is the acceptance case for guarding out-of-window rows in + cursor delete_insert models; see RUNBOOK.md, "Pending acceptance cases". + """, + tags [edge_case], + enabled false, + materialized incremental, + incremental_strategy delete_insert, + cursor delivered_on, + cursor_type timestamp, + cursor_grain day, + cursor_inputs ( + edge__delivery_updates ordered_on, + ), + columns ( + delivery_id (type VARCHAR, nullable false, audits [not_null, unique]), + shipment_id (type VARCHAR), + ordered_on (type DATE), + delivered_on (type DATE), + ), +); + +WITH updates AS ( + SELECT + u.delivery_id, + u.shipment_id, + u.ordered_on, + u.delivered_on + FROM __source("edge__delivery_updates") AS u +), + +final AS ( + SELECT + u.delivery_id, + u.shipment_id, + u.ordered_on, + u.delivered_on + FROM updates AS u + WHERE + u.delivered_on IS NOT NULL + AND u.ordered_on >= __cursor_start() + AND u.ordered_on < __cursor_end() +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql new file mode 100644 index 0000000000..0712d418b8 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_partner_listing_export.sql @@ -0,0 +1,30 @@ +MODEL ( + description """ + One row per partner listing in the partner's own feed format, with quoted camelCase column names + and prices in major units + """, + tags [edge_case], +); + +WITH listings AS ( + SELECT + d.listing_id, + d.unit_price_minor, + d.listing_status + FROM __ref("dim_edge_partner_listings") AS d +), + +final AS ( + SELECT + l.listing_id AS "listingId", + ( + l.unit_price_minor / 100 + )::NUMBER(18, 2) AS "unitPrice", + ( + l.listing_status = 'live' + ) AS "isLive" + FROM listings AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_payload_price_types.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_payload_price_types.sql new file mode 100644 index 0000000000..4e18e64588 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_payload_price_types.sql @@ -0,0 +1,42 @@ +MODEL ( + description """ + One row per published payload event, reading the typed values back out of the event document + built in another model: the kind Snowflake kept for each value and the values cast again + """, + tags [edge_case], + columns ( + message_id (type VARCHAR, nullable false, audits [not_null, unique]), + unit_price_kind (type VARCHAR), + unit_price (type FLOAT), + amount_kind (type VARCHAR), + amount (type NUMBER(18, 2)), + placed_on_kind (type VARCHAR), + placed_on (type DATE), + placed_on_text (type VARCHAR), + channel_kind (type VARCHAR), + ), +); + +WITH events AS ( + SELECT + e.message_id, + e.event_document + FROM __ref("fct_edge_payload_events") AS e +), + +final AS ( + SELECT + e.message_id, + TYPEOF(e.event_document:unit_price)::VARCHAR AS unit_price_kind, + e.event_document:unit_price::FLOAT AS unit_price, + TYPEOF(e.event_document:amount)::VARCHAR AS amount_kind, + e.event_document:amount::NUMBER(18, 2) AS amount, + TYPEOF(e.event_document:placed_on)::VARCHAR AS placed_on_kind, + e.event_document:placed_on::DATE AS placed_on, + e.event_document:placed_on::VARCHAR AS placed_on_text, + TYPEOF(e.event_document:channel)::VARCHAR AS channel_kind + FROM events AS e +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_listings.sql b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_listings.sql new file mode 100644 index 0000000000..b3a4cc0ac2 --- /dev/null +++ b/benchmarks/duckshop/project/models/staging/edge_cases/stg_edge_partner_listings.sql @@ -0,0 +1,28 @@ +MODEL ( + description """ + One row per partner listing, reading the feed's quoted camelCase columns by their exact names + and renaming them to snake case + """, + tags [edge_case], + columns ( + listing_id (type VARCHAR, nullable false, audits [not_null, unique]), + unit_price_minor (type NUMBER(38, 0)), + listing_status (type VARCHAR), + ), +); + +WITH src AS ( + SELECT * + FROM __source("edge__partner_listings") +), + +final AS ( + SELECT + s."listingId"::VARCHAR AS listing_id, + s."unitPriceMinor"::NUMBER(38, 0) AS unit_price_minor, + LOWER(s."listingStatus")::VARCHAR AS listing_status + FROM src AS s +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/python/loaders/_edge_cases.py b/benchmarks/duckshop/project/python/loaders/_edge_cases.py index 697d7b5f4a..310fef3ca4 100644 --- a/benchmarks/duckshop/project/python/loaders/_edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/_edge_cases.py @@ -284,6 +284,65 @@ def ticket_notes() -> Rows: ] +def partner_listings() -> Rows: + listings = [ + ("LS-MALLARD-01", 1299, "Live"), + ("LS-TEAL-02", 899, "OUT_OF_STOCK"), + ("ls-teal-02", 905, "live"), + ] + return [ + { + '"listingId"': listing, + '"unitPriceMinor"': price, + '"listingStatus"': status, + "_loaded_at": _at("2026-03-05 00:00:00"), + } + for listing, price, status in listings + ] + + +DELIVERY_BATCH_VAR = "edge_delivery_batch" +"""Project variable naming the last delivery-update load to include; "2" (all loads) by default.""" + + +def delivery_updates(batch: int = 2) -> Rows: + """The delivery change log as it stands after the given load.""" + + first = "2026-03-01 10:00:00" + second = "2026-03-02 10:00:00" + updates = [ + # load, shipment, order, status, ordered on, delivered on, op, seq + (first, "SH-1", "DS-O-3001", "label_created", "2026-02-27", None, "I", 10), + (first, "SH-1", "DS-O-3001", "in_transit", "2026-02-27", None, "U", 12), + (first, "SH-2", "DS-O-3002", "label_created", "2026-02-28", None, "I", 20), + (first, "SH-3", "DS-O-3003", "delivered", "2026-02-20", "2026-02-25", "I", 30), + (first, "SH-4", "DS-O-3004", "in_transit", "2026-02-26", None, "I", 40), + # A late version older than the one already loaded must not win. + (second, "SH-1", "DS-O-3001", "picked_up", "2026-02-27", None, "U", 11), + # The same version delivered again. + (second, "SH-2", "DS-O-3002", "label_created", "2026-02-28", None, "I", 20), + (second, "SH-2", "DS-O-3002", "delivered", "2026-02-28", "2026-03-02", "U", 21), + (second, "SH-4", "DS-O-3004", "in_transit", "2026-02-26", None, "D", 41), + (second, "SH-5", "DS-O-3005", "label_created", "2026-03-01", None, "I", 50), + ] + loads = [first] if batch == 1 else [first, second] + return [ + { + "delivery_id": f"DU-{index + 1:04d}", + "shipment_id": shipment, + "order_ref": order, + "delivery_status": status, + "ordered_on": ordered, + "delivered_on": delivered, + "_op": op, + "_seq": seq, + "_loaded_at": _at(loaded), + } + for index, (loaded, shipment, order, status, ordered, delivered, op, seq) in enumerate(updates) + if loaded in loads + ] + + TABLES: dict[str, Rows] = { "edge__partner_payloads": partner_payloads(), "edge__gateway_callbacks": gateway_callbacks(), @@ -291,4 +350,6 @@ def ticket_notes() -> Rows: "edge__customer_profiles": customer_profiles(), "edge__ledger_entries": ledger_entries(), "edge__ticket_notes": ticket_notes(), + "edge__partner_listings": partner_listings(), + "edge__delivery_updates": delivery_updates(), } diff --git a/benchmarks/duckshop/project/python/loaders/edge_cases.py b/benchmarks/duckshop/project/python/loaders/edge_cases.py index cb8af07a79..c00991b8ac 100644 --- a/benchmarks/duckshop/project/python/loaders/edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/edge_cases.py @@ -4,7 +4,7 @@ and they never touch the core tables, so core outputs and expectations stay the same. """ -from python.loaders._edge_cases import TABLES +from python.loaders._edge_cases import DELIVERY_BATCH_VAR, TABLES, delivery_updates from python.loaders._writer import statements from sqlbuild.executor.load.models import LoaderContext @@ -46,3 +46,17 @@ def edge__ledger_entries(ctx: LoaderContext) -> object: @loader def edge__ticket_notes(ctx: LoaderContext) -> object: return _load(ctx, "edge__ticket_notes") + + +@loader +def edge__partner_listings(ctx: LoaderContext) -> object: + return _load(ctx, "edge__partner_listings") + + +@loader +def edge__delivery_updates(ctx: LoaderContext) -> object: + batch = int(ctx.vars.get(DELIVERY_BATCH_VAR, 2)) # type: ignore[call-overload] + rows = delivery_updates(batch) + for sql in statements(ctx.destination, "edge__delivery_updates", rows): + ctx.execute_sql(sql) + return ctx.result(metadata={"rows": len(rows), "batch": batch}) diff --git a/benchmarks/duckshop/project/sources/edge_cases.yml b/benchmarks/duckshop/project/sources/edge_cases.yml index 6bb45d8c59..6b7b5795a1 100644 --- a/benchmarks/duckshop/project/sources/edge_cases.yml +++ b/benchmarks/duckshop/project/sources/edge_cases.yml @@ -210,3 +210,64 @@ sources: nullable: false audits: - not_null + - name: edge__partner_listings + description: >- + Partner listing snapshots whose feed names columns in quoted camelCase, which only quoted + references can read + managed: true + contract: enforced + columns: + - name: '"listingId"' + type: VARCHAR + nullable: false + - name: '"unitPriceMinor"' + type: NUMBER(38,0) + - name: '"listingStatus"' + type: VARCHAR + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null + - name: edge__delivery_updates + description: >- + Shipment delivery change log delivered in two loads, the second carrying late older versions, + a redelivered version, a delete and new versions + managed: true + contract: enforced + columns: + - name: delivery_id + type: VARCHAR + nullable: false + audits: + - not_null + - unique + - name: shipment_id + type: VARCHAR + nullable: false + audits: + - not_null + - name: order_ref + type: VARCHAR + - name: delivery_status + type: VARCHAR + - name: ordered_on + type: DATE + - name: delivered_on + type: DATE + - name: _op + type: VARCHAR + nullable: false + audits: + - accepted_values: + values: ["I", "U", "D"] + - name: _seq + type: NUMBER(38,0) + nullable: false + audits: + - not_null + - name: _loaded_at + type: TIMESTAMP_NTZ + nullable: false + audits: + - not_null diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 5338300a25..9dd0e90060 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -90,6 +90,9 @@ valuation_date = "2026-03-31" lookup_customer_id = "C000001" # Volume multiplier for the synthetic data loaders in python/loaders; 1 is the small default. data_scale = "1" +# Last load of the edge-case delivery change log to include; "1" replays the first load only (see +# RUNBOOK.md, "Edge cases"). +edge_delivery_batch = "2" [scenario.local_type_overrides.snowflake] "VARIANT" = "JSON" @@ -277,3 +280,21 @@ reason = """ The edge case selects the key columns that FULL OUTER JOIN ... USING merges. They belong to neither side, so qualifying them with either alias would drop the keys of rows found only on the other side. """ + +[[rules.rule_ignores]] +rules = ["SQBRTEST201"] +paths = ["models/marts/edge_cases/rpt_edge_partner_listing_export.sql"] +reason = """ +The partner feed export names its columns in quoted camelCase. A MODEL header cannot declare a +quoted column name, so the model has no columns block to attach audits to, and a generic audit that +names "listingId" fails to bind because the quoted name is folded to lower case. +""" + +[[rules.rule_ignores]] +rules = ["SQBRTEST202"] +paths = ["models/staging/edge_cases/stg_edge_partner_listings.sql"] +reason = """ +Unit-test and scenario fixtures for a source whose declared columns are quoted camelCase names are +rejected (B302 and S510), so this model is covered by building the edge-case area on Snowflake, +where it reads the loaded quoted columns, instead of by a unit test. +""" diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/scenario.json b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/scenario.json new file mode 100644 index 0000000000..2959c94a0c --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/scenario.json @@ -0,0 +1,68 @@ +{ + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-28T18:47:37Z", + "format": "jsonl", + "input_fingerprint": "9d9ce1a2ac3f9bb5cfb5400dc4d05daac103ad39b0393ed0933e5ae9aec11f80", + "relations": [ + { + "bytes": 1021, + "columns": [ + { + "local_type": "TEXT(5)", + "name": "delivery_id", + "warehouse_type": "VARCHAR(5)" + }, + { + "local_type": "TEXT(4)", + "name": "shipment_id", + "warehouse_type": "VARCHAR(4)" + }, + { + "local_type": "TEXT(9)", + "name": "order_ref", + "warehouse_type": "VARCHAR(9)" + }, + { + "local_type": "TEXT(13)", + "name": "delivery_status", + "warehouse_type": "VARCHAR(13)" + }, + { + "local_type": "DATE", + "name": "ordered_on", + "warehouse_type": "DATE" + }, + { + "local_type": "DATE", + "name": "delivered_on", + "warehouse_type": "DATE" + }, + { + "local_type": "TEXT(1)", + "name": "_op", + "warehouse_type": "VARCHAR(1)" + }, + { + "local_type": "DECIMAL(38, 0)", + "name": "_seq", + "warehouse_type": "NUMBER(38,0)" + }, + { + "local_type": "TIMESTAMP(9)", + "name": "_loaded_at", + "warehouse_type": "TIMESTAMP_NTZ(9)" + } + ], + "file": "sources/edge__delivery_updates.jsonl", + "kind": "source", + "logical_name": "edge__delivery_updates", + "row_count": 5 + } + ], + "scenario_name": "edge_delivery_status__cursor_model_over_far_apart_fixture_loads", + "sqlbuild_version": "0.184.0", + "total_bytes": 1021, + "total_rows": 5, + "version": 1 +} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/sources/edge__delivery_updates.jsonl b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/sources/edge__delivery_updates.jsonl new file mode 100644 index 0000000000..295f4ffdbc --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_delivery_status__cursor_model_over_far_apart_fixture_loads/sources/edge__delivery_updates.jsonl @@ -0,0 +1,5 @@ +{"DELIVERED_ON":null,"DELIVERY_ID":"DU-01","DELIVERY_STATUS":"label_created","ORDERED_ON":"1899-12-30","ORDER_REF":"DS-O-3001","SHIPMENT_ID":"SH-1","_LOADED_AT":"1900-01-01T00:00:00","_OP":"I","_SEQ":10} +{"DELIVERED_ON":"2999-12-29","DELIVERY_ID":"DU-02","DELIVERY_STATUS":"delivered","ORDERED_ON":"1899-12-30","ORDER_REF":"DS-O-3001","SHIPMENT_ID":"SH-1","_LOADED_AT":"2999-12-30T00:00:00","_OP":"U","_SEQ":12} +{"DELIVERED_ON":null,"DELIVERY_ID":"DU-03","DELIVERY_STATUS":"in_transit","ORDERED_ON":"1899-12-30","ORDER_REF":"DS-O-3001","SHIPMENT_ID":"SH-1","_LOADED_AT":"2999-12-30T00:00:00","_OP":"U","_SEQ":11} +{"DELIVERED_ON":null,"DELIVERY_ID":"DU-04","DELIVERY_STATUS":"label_created","ORDERED_ON":"2026-03-01","ORDER_REF":"DS-O-3002","SHIPMENT_ID":"SH-2","_LOADED_AT":"1900-01-01T00:00:00","_OP":"I","_SEQ":20} +{"DELIVERED_ON":null,"DELIVERY_ID":"DU-05","DELIVERY_STATUS":"label_created","ORDERED_ON":"2026-03-01","ORDER_REF":"DS-O-3002","SHIPMENT_ID":"SH-2","_LOADED_AT":"2999-12-30T00:00:00","_OP":"D","_SEQ":21} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/refs/stg_edge_partner_listings.jsonl b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/refs/stg_edge_partner_listings.jsonl new file mode 100644 index 0000000000..11aaea9a2d --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/refs/stg_edge_partner_listings.jsonl @@ -0,0 +1,3 @@ +{"LISTING_ID":"LS-MALLARD-01","LISTING_STATUS":"live","UNIT_PRICE_MINOR":1299} +{"LISTING_ID":"LS-TEAL-02","LISTING_STATUS":"out_of_stock","UNIT_PRICE_MINOR":899} +{"LISTING_ID":"ls-teal-02","LISTING_STATUS":null,"UNIT_PRICE_MINOR":905} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/scenario.json b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/scenario.json new file mode 100644 index 0000000000..f7af4efb1b --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_partner_listings__star_contract_and_quoted_export/scenario.json @@ -0,0 +1,38 @@ +{ + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-28T18:47:37Z", + "format": "jsonl", + "input_fingerprint": "a447089b96c70f06bdd4d5377ba1b89f833e97f935d5b380178620eaa9d0ea37", + "relations": [ + { + "bytes": 235, + "columns": [ + { + "local_type": "TEXT(13)", + "name": "listing_id", + "warehouse_type": "VARCHAR(13)" + }, + { + "local_type": "DECIMAL(38, 0)", + "name": "unit_price_minor", + "warehouse_type": "NUMBER(38,0)" + }, + { + "local_type": "TEXT(12)", + "name": "listing_status", + "warehouse_type": "VARCHAR(12)" + } + ], + "file": "refs/stg_edge_partner_listings.jsonl", + "kind": "ref", + "logical_name": "stg_edge_partner_listings", + "row_count": 3 + } + ], + "scenario_name": "edge_partner_listings__star_contract_and_quoted_export", + "sqlbuild_version": "0.184.0", + "total_bytes": 235, + "total_rows": 3, + "version": 1 +} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/scenario.json b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/scenario.json new file mode 100644 index 0000000000..e492c80709 --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/scenario.json @@ -0,0 +1,58 @@ +{ + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-28T18:47:37Z", + "format": "jsonl", + "input_fingerprint": "baf56e87956fd7611a41783999279faeb360c90655f9acbb3720e5574cb4055b", + "relations": [ + { + "bytes": 570, + "columns": [ + { + "local_type": "TEXT(4)", + "name": "delivery_id", + "warehouse_type": "VARCHAR(4)" + }, + { + "local_type": "TEXT(4)", + "name": "message_id", + "warehouse_type": "VARCHAR(4)" + }, + { + "local_type": "TEXT(7)", + "name": "partner_code", + "warehouse_type": "VARCHAR(7)" + }, + { + "local_type": "TIMESTAMP(9)", + "name": "received_at", + "warehouse_type": "TIMESTAMP_NTZ(9)" + }, + { + "local_type": "JSON", + "name": "payload", + "warehouse_type": "VARIANT" + }, + { + "local_type": "TEXT(16777216)", + "name": "payload_text", + "warehouse_type": "VARCHAR(16777216)" + }, + { + "local_type": "TIMESTAMP(9)", + "name": "_loaded_at", + "warehouse_type": "TIMESTAMP_NTZ(9)" + } + ], + "file": "sources/edge__partner_payloads.jsonl", + "kind": "source", + "logical_name": "edge__partner_payloads", + "row_count": 2 + } + ], + "scenario_name": "edge_payload_lines__mixed_type_arrays_flatten_to_lines", + "sqlbuild_version": "0.184.0", + "total_bytes": 570, + "total_rows": 2, + "version": 1 +} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/sources/edge__partner_payloads.jsonl b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/sources/edge__partner_payloads.jsonl new file mode 100644 index 0000000000..3292374cb3 --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_payload_lines__mixed_type_arrays_flatten_to_lines/sources/edge__partner_payloads.jsonl @@ -0,0 +1,2 @@ +{"DELIVERY_ID":"DL-1","MESSAGE_ID":"PM-1","PARTNER_CODE":"mkt_eu","PAYLOAD":"{\n \"channel\": \"web\",\n \"lines\": [\n 1,\n {\n \"sku\": \"MALLARD-01\"\n }\n ]\n}","PAYLOAD_TEXT":null,"RECEIVED_AT":null,"_LOADED_AT":"2026-03-01T10:00:00"} +{"DELIVERY_ID":"DL-2","MESSAGE_ID":"PM-2","PARTNER_CODE":"giftbox","PAYLOAD":"{\n \"lines\": [\n {\n \"sku\": \"TEAL-02\",\n \"y\": [\n 1,\n {\n \"z\": 2\n }\n ]\n },\n null\n ]\n}","PAYLOAD_TEXT":null,"RECEIVED_AT":null,"_LOADED_AT":"2026-03-01T10:00:00"} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/scenario.json b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/scenario.json new file mode 100644 index 0000000000..11ca8d3ad3 --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/scenario.json @@ -0,0 +1,58 @@ +{ + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-28T18:47:37Z", + "format": "jsonl", + "input_fingerprint": "197e68dc3cc4c0371b0796d66e0181b98f8578056fb5060dfe5d25bd86f9a1c0", + "relations": [ + { + "bytes": 663, + "columns": [ + { + "local_type": "TEXT(4)", + "name": "delivery_id", + "warehouse_type": "VARCHAR(4)" + }, + { + "local_type": "TEXT(4)", + "name": "message_id", + "warehouse_type": "VARCHAR(4)" + }, + { + "local_type": "TEXT(7)", + "name": "partner_code", + "warehouse_type": "VARCHAR(7)" + }, + { + "local_type": "TIMESTAMP(9)", + "name": "received_at", + "warehouse_type": "TIMESTAMP_NTZ(9)" + }, + { + "local_type": "JSON", + "name": "payload", + "warehouse_type": "VARIANT" + }, + { + "local_type": "TEXT(16777216)", + "name": "payload_text", + "warehouse_type": "VARCHAR(16777216)" + }, + { + "local_type": "TIMESTAMP(9)", + "name": "_loaded_at", + "warehouse_type": "TIMESTAMP_NTZ(9)" + } + ], + "file": "sources/edge__partner_payloads.jsonl", + "kind": "source", + "logical_name": "edge__partner_payloads", + "row_count": 3 + } + ], + "scenario_name": "edge_variant_chain__typed_payload_survives_models", + "sqlbuild_version": "0.184.0", + "total_bytes": 663, + "total_rows": 3, + "version": 1 +} diff --git a/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/sources/edge__partner_payloads.jsonl b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/sources/edge__partner_payloads.jsonl new file mode 100644 index 0000000000..fa5945bce7 --- /dev/null +++ b/benchmarks/duckshop/project/tests/_scenario_snapshots/edge_variant_chain__typed_payload_survives_models/sources/edge__partner_payloads.jsonl @@ -0,0 +1,3 @@ +{"DELIVERY_ID":"DL-1","MESSAGE_ID":"PM-1","PARTNER_CODE":"mkt_eu","PAYLOAD":"{\n \"amount\": 19.99,\n \"channel\": \"web\",\n \"placed_on\": \"2026-03-01\"\n}","PAYLOAD_TEXT":null,"RECEIVED_AT":null,"_LOADED_AT":"2026-03-01T10:00:00"} +{"DELIVERY_ID":"DL-2","MESSAGE_ID":"PM-2","PARTNER_CODE":"mkt_eu","PAYLOAD":"{\n \"amount\": 0,\n \"channel\": null\n}","PAYLOAD_TEXT":null,"RECEIVED_AT":null,"_LOADED_AT":"2026-03-01T10:00:00"} +{"DELIVERY_ID":"DL-3","MESSAGE_ID":"PM-3","PARTNER_CODE":"giftbox","PAYLOAD":"{\n \"amount\": 0.30000000000000004,\n \"placed_on\": \"2028-02-28\"\n}","PAYLOAD_TEXT":null,"RECEIVED_AT":null,"_LOADED_AT":"2026-03-01T10:00:00"} diff --git a/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_delivery_status__cursor_model_over_far_apart_fixture_loads.sql b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_delivery_status__cursor_model_over_far_apart_fixture_loads.sql new file mode 100644 index 0000000000..94cbc68ca3 --- /dev/null +++ b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_delivery_status__cursor_model_over_far_apart_fixture_loads.sql @@ -0,0 +1,61 @@ +SCENARIO ( + description """ + The incremental delivery status, whose window comes from its cursor, builds over fixture loads a + millennium apart and keeps the latest version of each shipment, as the full-history rebuild does + """, + tags ["edge_case"], +); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '1899-12-30', NULL, 'I', 10, '1900-01-01 00:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'delivered', '1899-12-30', '2999-12-29', 'U', 12, '2999-12-30 00:00:00'), + ('DU-03', 'SH-1', 'DS-O-3001', 'in_transit', '1899-12-30', NULL, 'U', 11, '2999-12-30 00:00:00'), + ('DU-04', 'SH-2', 'DS-O-3002', 'label_created', '2026-03-01', NULL, 'I', 20, '1900-01-01 00:00:00'), + ('DU-05', 'SH-2', 'DS-O-3002', 'label_created', '2026-03-01', NULL, 'D', 21, '2999-12-30 00:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +expected_status AS ( + SELECT + v.shipment_id, + v.delivery_status, + v.delivered_on::DATE AS delivered_on, + v.is_deleted, + v.latest_seq::NUMBER(38, 0) AS latest_seq, + v.source_loaded_at::TIMESTAMP_NTZ AS source_loaded_at + FROM VALUES + ('SH-1', 'delivered', '2999-12-29', FALSE, 12, '2999-12-30 00:00:00'), + ('SH-2', 'label_created', NULL, TRUE, 21, '2999-12-30 00:00:00') AS v(shipment_id, delivery_status, delivered_on, is_deleted, latest_seq, source_loaded_at) +), + +__expected__int_edge_delivery_status AS ( + SELECT + e.shipment_id, + e.delivery_status, + e.delivered_on, + e.is_deleted, + e.latest_seq, + e.source_loaded_at + FROM expected_status AS e +), + +__expected__int_edge_delivery_status_rebuilt AS ( + SELECT + e.shipment_id, + e.delivery_status, + e.delivered_on, + e.is_deleted, + e.latest_seq, + e.source_loaded_at + FROM expected_status AS e +) diff --git a/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_partner_listings__star_contract_and_quoted_export.sql b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_partner_listings__star_contract_and_quoted_export.sql new file mode 100644 index 0000000000..f99b0af079 --- /dev/null +++ b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_partner_listings__star_contract_and_quoted_export.sql @@ -0,0 +1,40 @@ +SCENARIO ( + description """ + Partner listings are published under a lower-case contract through a star, whose columns + Snowflake names in upper case, and written back out in the partner's quoted camelCase + """, + tags ["edge_case"], +); + +WITH __ref__stg_edge_partner_listings AS ( + SELECT + v.listing_id, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + v.listing_status + FROM VALUES + ('LS-MALLARD-01', 1299, 'live'), + ('LS-TEAL-02', 899, 'out_of_stock'), + ('ls-teal-02', 905, NULL) AS v(listing_id, unit_price_minor, listing_status) +), + +__expected__dim_edge_partner_listings AS ( + SELECT + v.listing_id, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + v.listing_status + FROM VALUES + ('LS-MALLARD-01', 1299, 'live'), + ('LS-TEAL-02', 899, 'out_of_stock'), + ('ls-teal-02', 905, NULL) AS v(listing_id, unit_price_minor, listing_status) +), + +__expected__rpt_edge_partner_listing_export AS ( + SELECT + v.listing_id AS "listingId", + v.unit_price::NUMBER(18, 2) AS "unitPrice", + v.is_live AS "isLive" + FROM VALUES + ('LS-MALLARD-01', 12.99, TRUE), + ('LS-TEAL-02', 8.99, FALSE), + ('ls-teal-02', 9.05, NULL) AS v(listing_id, unit_price, is_live) +) diff --git a/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_payload_lines__mixed_type_arrays_flatten_to_lines.sql b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_payload_lines__mixed_type_arrays_flatten_to_lines.sql new file mode 100644 index 0000000000..65b0b45129 --- /dev/null +++ b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_payload_lines__mixed_type_arrays_flatten_to_lines.sql @@ -0,0 +1,69 @@ +SCENARIO ( + description """ + Partner payloads whose line arrays mix numbers, objects and JSON nulls flatten to one row per + line, and every output column of the published event documents is compared, VARIANT included + """, + tags ["edge_case"], + expected_columns "all", +); + +WITH __source__edge__partner_payloads AS ( + SELECT + v.delivery_id, + v.message_id, + v.partner_code, + NULL::TIMESTAMP_NTZ AS received_at, + PARSE_JSON(v.payload) AS payload, + NULL::VARCHAR AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DL-1', 'PM-1', 'mkt_eu', '{"lines": [1, {"sku": "MALLARD-01"}], "channel": "web"}'), + ('DL-2', 'PM-2', 'giftbox', '{"lines": [{"sku": "TEAL-02", "y": [1, {"z": 2}]}, null]}') AS v(delivery_id, message_id, partner_code, payload) +), + +__expected__int_edge_payload_lines AS ( + SELECT + v.message_id, + v.line_index, + v.line_key, + v.line_path, + v.line_kind, + v.line_sku + FROM VALUES + ('PM-1', 0, NULL, '[0]', 'INTEGER', NULL), + ('PM-1', 1, NULL, '[1]', 'OBJECT', 'MALLARD-01'), + ('PM-2', 0, NULL, '[0]', 'OBJECT', 'TEAL-02'), + ('PM-2', 1, NULL, '[1]', 'NULL_VALUE', NULL) AS v(message_id, line_index, line_key, line_path, line_kind, line_sku) +), + +__expected__fct_edge_payload_events AS ( + SELECT + 'PM-1' AS message_id, + 'MKT_EU' AS partner_code, + OBJECT_CONSTRUCT_KEEP_NULL( + 'channel', + 'web', + 'unit_price', + NULL::FLOAT, + 'amount', + NULL::NUMBER(18, 2), + 'placed_on', + NULL::DATE + )::VARIANT AS event_document, + '{"amount":null,"channel":"web","placed_on":null,"unit_price":null}' AS event_document_text + UNION ALL + SELECT + 'PM-2', + 'GIFTBOX', + OBJECT_CONSTRUCT_KEEP_NULL( + 'channel', + NULL::VARCHAR, + 'unit_price', + NULL::FLOAT, + 'amount', + NULL::NUMBER(18, 2), + 'placed_on', + NULL::DATE + )::VARIANT, + '{"amount":null,"channel":null,"placed_on":null,"unit_price":null}' +) diff --git a/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_variant_chain__typed_payload_survives_models.sql b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_variant_chain__typed_payload_survives_models.sql new file mode 100644 index 0000000000..019939d982 --- /dev/null +++ b/benchmarks/duckshop/project/tests/scenarios/edge_cases/edge_variant_chain__typed_payload_survives_models.sql @@ -0,0 +1,45 @@ +SCENARIO ( + description """ + Event documents built from typed SQL values in one model keep each value's type when another + model reads them back: a FLOAT price, an exact amount, a date and JSON nulls + """, + tags ["edge_case"], +); + +WITH __source__edge__partner_payloads AS ( + SELECT + v.delivery_id, + v.message_id, + v.partner_code, + NULL::TIMESTAMP_NTZ AS received_at, + PARSE_JSON(v.payload) AS payload, + NULL::VARCHAR AS payload_text, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DL-1', 'PM-1', 'mkt_eu', '{"channel": "web", "amount": 19.99, "placed_on": "2026-03-01"}'), + ('DL-2', 'PM-2', 'mkt_eu', '{"channel": null, "amount": -0.0}'), + ('DL-3', 'PM-3', 'giftbox', '{"amount": 0.30000000000000004, "placed_on": "2028-02-28"}') AS v(delivery_id, message_id, partner_code, payload) +), + +__expected__fct_edge_payload_events AS ( + SELECT + v.message_id, + v.event_document_text + FROM VALUES + ('PM-1', '{"amount":19.99,"channel":"web","placed_on":"2026-03-01","unit_price":1.999000000000000e+01}'), + ('PM-2', '{"amount":0,"channel":null,"placed_on":null,"unit_price":0.000000000000000e+00}'), + ('PM-3', '{"amount":0.3,"channel":null,"placed_on":"2028-02-28","unit_price":3.000000000000000e-01}') AS v(message_id, event_document_text) +), + +__expected__rpt_edge_payload_price_types AS ( + SELECT + v.message_id, + v.unit_price_kind, + v.amount_kind, + v.placed_on_kind, + v.channel_kind + FROM VALUES + ('PM-1', 'DOUBLE', 'DECIMAL', 'DATE', 'VARCHAR'), + ('PM-2', 'DOUBLE', 'DECIMAL', 'NULL_VALUE', 'NULL_VALUE'), + ('PM-3', 'DOUBLE', 'DECIMAL', 'DATE', 'NULL_VALUE') AS v(message_id, unit_price_kind, amount_kind, placed_on_kind, channel_kind) +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status.sql new file mode 100644 index 0000000000..8b527fee49 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status.sql @@ -0,0 +1,151 @@ +TEST (name "int_edge_delivery_status__keeps_the_latest_version_of_every_shipment"); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, 'I', 10, '2026-03-01 10:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, 'U', 12, '2026-03-01 10:00:00'), + ('DU-03', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', 'I', 30, '2026-03-01 10:00:00'), + ('DU-05', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'I', 40, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, 'U', 11, '2026-03-02 10:00:00'), + ('DU-07', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', 'U', 21, '2026-03-02 10:00:00'), + ('DU-09', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'D', 41, '2026-03-02 10:00:00'), + ('DU-10', 'SH-5', 'DS-O-3005', 'label_created', '2026-03-01', NULL, 'I', 50, '2026-03-02 10:00:00'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, 'I', 60, '2026-03-03 10:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +__expected__int_edge_delivery_status AS ( + SELECT + 'SH-1' AS shipment_id, + 'DS-O-3001' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 12 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-2' AS shipment_id, + 'DS-O-3002' AS order_ref, + 'delivered' AS delivery_status, + '2026-03-02'::DATE AS delivered_on, + FALSE AS is_deleted, + 21 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-3' AS shipment_id, + 'DS-O-3003' AS order_ref, + 'delivered' AS delivery_status, + '2026-02-25'::DATE AS delivered_on, + FALSE AS is_deleted, + 30 AS latest_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-4' AS shipment_id, + 'DS-O-3004' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + TRUE AS is_deleted, + 41 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-5' AS shipment_id, + 'DS-O-3005' AS order_ref, + 'label_created' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 50 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-6' AS shipment_id, + 'DS-O-3006' AS order_ref, + 'label_created' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 60 AS latest_seq, + '2026-03-03 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at +) + +TEST ( + name "int_edge_delivery_status__recomputes_shipments_touched_in_the_window_from_history", + cursor_start "2026-03-02 00:00:00", + cursor_end "2026-03-03 00:00:00", +); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, 'I', 10, '2026-03-01 10:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, 'U', 12, '2026-03-01 10:00:00'), + ('DU-03', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', 'I', 30, '2026-03-01 10:00:00'), + ('DU-05', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'I', 40, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, 'U', 11, '2026-03-02 10:00:00'), + ('DU-07', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', 'U', 21, '2026-03-02 10:00:00'), + ('DU-09', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'D', 41, '2026-03-02 10:00:00'), + ('DU-10', 'SH-5', 'DS-O-3005', 'label_created', '2026-03-01', NULL, 'I', 50, '2026-03-02 10:00:00'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, 'I', 60, '2026-03-03 10:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +__expected__int_edge_delivery_status AS ( + SELECT + 'SH-1' AS shipment_id, + 'DS-O-3001' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 12 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-2' AS shipment_id, + 'DS-O-3002' AS order_ref, + 'delivered' AS delivery_status, + '2026-03-02'::DATE AS delivered_on, + FALSE AS is_deleted, + 21 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-4' AS shipment_id, + 'DS-O-3004' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + TRUE AS is_deleted, + 41 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-5' AS shipment_id, + 'DS-O-3005' AS order_ref, + 'label_created' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 50 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_rebuilt.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_rebuilt.sql new file mode 100644 index 0000000000..8f4012856f --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_rebuilt.sql @@ -0,0 +1,82 @@ +TEST (name "int_edge_delivery_status_rebuilt__keeps_the_latest_version_of_every_shipment"); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, 'I', 10, '2026-03-01 10:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, 'U', 12, '2026-03-01 10:00:00'), + ('DU-03', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', 'I', 30, '2026-03-01 10:00:00'), + ('DU-05', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'I', 40, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, 'U', 11, '2026-03-02 10:00:00'), + ('DU-07', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', 'U', 21, '2026-03-02 10:00:00'), + ('DU-09', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'D', 41, '2026-03-02 10:00:00'), + ('DU-10', 'SH-5', 'DS-O-3005', 'label_created', '2026-03-01', NULL, 'I', 50, '2026-03-02 10:00:00'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, 'I', 60, '2026-03-03 10:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +__expected__int_edge_delivery_status_rebuilt AS ( + SELECT + 'SH-1' AS shipment_id, + 'DS-O-3001' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 12 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-2' AS shipment_id, + 'DS-O-3002' AS order_ref, + 'delivered' AS delivery_status, + '2026-03-02'::DATE AS delivered_on, + FALSE AS is_deleted, + 21 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-3' AS shipment_id, + 'DS-O-3003' AS order_ref, + 'delivered' AS delivery_status, + '2026-02-25'::DATE AS delivered_on, + FALSE AS is_deleted, + 30 AS latest_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-4' AS shipment_id, + 'DS-O-3004' AS order_ref, + 'in_transit' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + TRUE AS is_deleted, + 41 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-5' AS shipment_id, + 'DS-O-3005' AS order_ref, + 'label_created' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 50 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-6' AS shipment_id, + 'DS-O-3006' AS order_ref, + 'label_created' AS delivery_status, + CAST(NULL AS DATE) AS delivered_on, + FALSE AS is_deleted, + 60 AS latest_seq, + '2026-03-03 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_update_versions.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_update_versions.sql new file mode 100644 index 0000000000..11cefd5c3a --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_update_versions.sql @@ -0,0 +1,116 @@ +TEST (name "int_edge_delivery_update_versions__keeps_every_version_of_every_load"); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, 'I', 10, '2026-03-01 10:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, 'U', 12, '2026-03-01 10:00:00'), + ('DU-03', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', 'I', 30, '2026-03-01 10:00:00'), + ('DU-05', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'I', 40, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, 'U', 11, '2026-03-02 10:00:00'), + ('DU-07', 'SH-2', 'DS-O-3002', 'label_created', '2026-02-28', NULL, 'I', 20, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', 'U', 21, '2026-03-02 10:00:00'), + ('DU-09', 'SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, 'D', 41, '2026-03-02 10:00:00'), + ('DU-10', 'SH-5', 'DS-O-3005', 'label_created', '2026-03-01', NULL, 'I', 50, '2026-03-02 10:00:00'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, 'I', 60, '2026-03-03 10:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +__expected__int_edge_delivery_update_versions AS ( + SELECT + 'DU-01' AS delivery_id, + 'SH-1' AS shipment_id, + 'label_created' AS delivery_status, + FALSE AS is_delete, + 10 AS version_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-02' AS delivery_id, + 'SH-1' AS shipment_id, + 'in_transit' AS delivery_status, + FALSE AS is_delete, + 12 AS version_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-03' AS delivery_id, + 'SH-2' AS shipment_id, + 'label_created' AS delivery_status, + FALSE AS is_delete, + 20 AS version_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-04' AS delivery_id, + 'SH-3' AS shipment_id, + 'delivered' AS delivery_status, + FALSE AS is_delete, + 30 AS version_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-05' AS delivery_id, + 'SH-4' AS shipment_id, + 'in_transit' AS delivery_status, + FALSE AS is_delete, + 40 AS version_seq, + '2026-03-01 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-06' AS delivery_id, + 'SH-1' AS shipment_id, + 'picked_up' AS delivery_status, + FALSE AS is_delete, + 11 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-07' AS delivery_id, + 'SH-2' AS shipment_id, + 'label_created' AS delivery_status, + FALSE AS is_delete, + 20 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-08' AS delivery_id, + 'SH-2' AS shipment_id, + 'delivered' AS delivery_status, + FALSE AS is_delete, + 21 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-09' AS delivery_id, + 'SH-4' AS shipment_id, + 'in_transit' AS delivery_status, + TRUE AS is_delete, + 41 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-10' AS delivery_id, + 'SH-5' AS shipment_id, + 'label_created' AS delivery_status, + FALSE AS is_delete, + 50 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at + UNION ALL + SELECT + 'DU-11' AS delivery_id, + 'SH-6' AS shipment_id, + 'label_created' AS delivery_status, + FALSE AS is_delete, + 60 AS version_seq, + '2026-03-03 10:00:00'::TIMESTAMP_NTZ(9) AS loaded_at +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql index b7bca8d240..309efecb75 100644 --- a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_order_line_labels.sql @@ -72,7 +72,7 @@ __expected__int_edge_order_line_labels AS ( 1 AS modal_quantity ) -TEST (name "int_edge_order_line_labels__picks_one_of_the_tied_lines_for_max_by_and_mode"); +TEST (name "int_edge_order_line_labels__picks_one_of_the_tied_lines_for_max_by_mode_and_any_value"); WITH __ref__stg_edge_ledger_entries AS ( SELECT @@ -92,12 +92,12 @@ WITH __ref__stg_edge_ledger_entries AS ( NULL AS fx_rate, NULL AS status, NULL AS reason_code, - NULL AS customer_tier + v.customer_tier FROM VALUES - ('LE-01', 'DS-O-1', 1, 'MALLARD-01', 1, 1299), - ('LE-02', 'DS-O-1', 2, 'TEAL-02', 2, 1299), - ('LE-03', 'DS-O-1', 3, 'EIDER-03', 1, 350), - ('LE-04', 'DS-O-1', 4, 'PINTAIL-04', 2, 350) AS v(entry_id, order_ref, line_no, sku, quantity, unit_price_minor) + ('LE-01', 'DS-O-1', 1, 'MALLARD-01', 1, 1299, 'Mallard'), + ('LE-02', 'DS-O-1', 2, 'TEAL-02', 2, 1299, 'Duckling'), + ('LE-03', 'DS-O-1', 3, 'EIDER-03', 1, 350, 'Mallard'), + ('LE-04', 'DS-O-1', 4, 'PINTAIL-04', 2, 350, 'Duckling') AS v(entry_id, order_ref, line_no, sku, quantity, unit_price_minor, customer_tier) ), __expected__int_edge_order_line_labels AS ( @@ -115,4 +115,5 @@ __assert__tied_picks_come_from_the_tied_lines AS ( l.priciest_sku NOT IN ('MALLARD-01', 'TEAL-02') OR l.cheapest_sku NOT IN ('EIDER-03', 'PINTAIL-04') OR l.modal_quantity NOT IN (1, 2) + OR l.customer_tier NOT IN ('Mallard', 'Duckling') ) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_dim_edge_partner_listings.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_dim_edge_partner_listings.sql new file mode 100644 index 0000000000..7a0b3998ca --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_dim_edge_partner_listings.sql @@ -0,0 +1,23 @@ +TEST (name "dim_edge_partner_listings__publishes_a_star_under_a_lower_case_contract"); + +WITH __ref__stg_edge_partner_listings AS ( + SELECT + v.listing_id, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + v.listing_status + FROM VALUES + ('LS-MALLARD-01', 1299, 'live'), + ('LS-TEAL-02', 899, 'out_of_stock') AS v(listing_id, unit_price_minor, listing_status) +), + +__expected__dim_edge_partner_listings AS ( + SELECT + 'LS-MALLARD-01' AS listing_id, + 1299 AS unit_price_minor, + 'live' AS listing_status + UNION ALL + SELECT + 'LS-TEAL-02' AS listing_id, + 899 AS unit_price_minor, + 'out_of_stock' AS listing_status +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_delivery_events.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_delivery_events.sql new file mode 100644 index 0000000000..aa691604d2 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_fct_edge_delivery_events.sql @@ -0,0 +1,35 @@ +TEST ( + name "fct_edge_delivery_events__emits_deliveries_outside_the_window_it_deletes", + cursor_start "2026-02-27", + cursor_end "2026-02-28", +); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + NULL AS order_ref, + NULL AS delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + 'I' AS _op, + v._seq::NUMBER(38, 0) AS _seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', '2026-02-27', '2026-02-27', 1), + ('DU-02', 'SH-2', '2026-02-27', '2026-03-04', 2), + ('DU-03', 'SH-3', '2026-02-26', '2026-02-27', 3), + ('DU-04', 'SH-4', '2026-02-27', NULL, 4) AS v(delivery_id, shipment_id, ordered_on, delivered_on, _seq) +), + +__expected__fct_edge_delivery_events AS ( + SELECT + 'DU-01' AS delivery_id, + '2026-02-27'::DATE AS ordered_on, + '2026-02-27'::DATE AS delivered_on + UNION ALL + SELECT + 'DU-02' AS delivery_id, + '2026-02-27'::DATE AS ordered_on, + '2026-03-04'::DATE AS delivered_on +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql new file mode 100644 index 0000000000..5660a2b3b7 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql @@ -0,0 +1,23 @@ +TEST (name "rpt_edge_partner_listing_export__writes_the_feed_in_quoted_camel_case"); + +WITH __ref__dim_edge_partner_listings AS ( + SELECT + v.listing_id, + v.unit_price_minor::NUMBER(38, 0) AS unit_price_minor, + v.listing_status + FROM VALUES + ('LS-MALLARD-01', 1299, 'live'), + ('LS-TEAL-02', 899, 'out_of_stock'), + ('LS-EIDER-03', 5, NULL) AS v(listing_id, unit_price_minor, listing_status) +), + +__expected__rpt_edge_partner_listing_export AS ( + SELECT + v.listing_id AS "listingId", + v.unit_price AS "unitPrice", + v.is_live AS "isLive" + FROM VALUES + ('LS-MALLARD-01', 12.99, TRUE), + ('LS-TEAL-02', 8.99, FALSE), + ('LS-EIDER-03', 0.05, NULL) AS v(listing_id, unit_price, is_live) +) diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_payload_price_types.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_payload_price_types.sql new file mode 100644 index 0000000000..efe985ed31 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_payload_price_types.sql @@ -0,0 +1,56 @@ +TEST (name "rpt_edge_payload_price_types__reads_typed_values_back_out_of_event_documents"); + +WITH __ref__fct_edge_payload_events AS ( + SELECT + v.message_id, + NULL AS partner_code, + OBJECT_CONSTRUCT_KEEP_NULL( + 'channel', + v.channel, + 'unit_price', + v.unit_price::FLOAT, + 'amount', + v.amount::NUMBER(18, 2), + 'placed_on', + v.placed_on::DATE + )::VARIANT AS event_document + FROM VALUES + ('PM-1', 'web', 19.99, 19.99, '2026-03-01'), + ('PM-2', NULL, -0.0, 0, NULL), + ('PM-3', 'app', 0.30000000000000004, 1234.5, '2028-02-28') AS v(message_id, channel, unit_price, amount, placed_on) +), + +__expected__rpt_edge_payload_price_types AS ( + SELECT + 'PM-1' AS message_id, + 'DOUBLE' AS unit_price_kind, + '19.99'::FLOAT AS unit_price, + 'DECIMAL' AS amount_kind, + 19.99::NUMBER(18, 2) AS amount, + 'DATE' AS placed_on_kind, + '2026-03-01'::DATE AS placed_on, + '2026-03-01' AS placed_on_text, + 'VARCHAR' AS channel_kind + UNION ALL + SELECT + 'PM-2' AS message_id, + 'DOUBLE' AS unit_price_kind, + '0.0'::FLOAT AS unit_price, + 'DECIMAL' AS amount_kind, + 0.00::NUMBER(18, 2) AS amount, + 'NULL_VALUE' AS placed_on_kind, + CAST(NULL AS DATE) AS placed_on, + CAST(NULL AS VARCHAR) AS placed_on_text, + 'NULL_VALUE' AS channel_kind + UNION ALL + SELECT + 'PM-3' AS message_id, + 'DOUBLE' AS unit_price_kind, + '0.30000000000000004'::FLOAT AS unit_price, + 'DECIMAL' AS amount_kind, + 1234.50::NUMBER(18, 2) AS amount, + 'DATE' AS placed_on_kind, + '2028-02-28'::DATE AS placed_on, + '2028-02-28' AS placed_on_text, + 'VARCHAR' AS channel_kind +) From 4234eafd05ebae06048da82664d6f5dd9d59b433 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:33:37 +0000 Subject: [PATCH 5/6] test(benchmarks): keep the DuckShop edge-case area out of core runs and document it Add scripts/core_project.py, which copies the project without the edge-case area for benchmarks; the copy compiles to the same files as the project before the area existed, and an E2E guard compiles it. The construct census counts the core project by default and takes --area. The loader checks cover the core sources only, with separate checks that the edge rows match their sources, ignore data_scale and replay the delivery change log as two loads. Add scripts/capture_edge_expectations.py, which captures edge unit-test expectations from Snowflake on the edge target. Document the area in the README, and in the RUNBOOK: excluding it from core runs, running and capturing it on its pinned session, the two-load replay, the pending acceptance case, and the SQLBuild limitations it found. --- benchmarks/duckshop/README.md | 72 ++++- benchmarks/duckshop/RUNBOOK.md | 122 ++++++-- .../project/python/loaders/_edge_cases.py | 263 ++++++++++++++--- .../project/sqlbuild_local.toml.example | 13 + .../test_rpt_edge_partner_listing_export.sql | 20 +- .../scripts/capture_edge_expectations.py | 278 ++++++++++++++++++ benchmarks/duckshop/scripts/census.py | 20 +- benchmarks/duckshop/scripts/core_project.py | 43 +++ .../duckshop/tests/test_synthetic_data.py | 78 ++++- .../cli/commands/main/compile/_test_types.py | 11 + .../test_duckshop_benchmark_compile.py | 41 +++ 11 files changed, 890 insertions(+), 71 deletions(-) create mode 100644 benchmarks/duckshop/scripts/capture_edge_expectations.py create mode 100644 benchmarks/duckshop/scripts/core_project.py diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 9d1705c3b0..745778dafd 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -19,7 +19,8 @@ Everything in it is synthetic: order references use a `DS-O-` prefix, email addr ```text benchmarks/duckshop/ ├── project/ the SQLBuild project (Snowflake adapter), with its data loaders in python/loaders/ -├── scripts/ tooling that is not part of the project: the construct census and cleanup SQL +├── scripts/ tooling that is not part of the project: the construct census, cleanup SQL, the +│ core-only copy and edge-case expectation capture ├── tests/ offline checks for the synthetic data loaders └── RUNBOOK.md loading, building, testing and capturing on Snowflake, and cleaning up ``` @@ -39,6 +40,8 @@ benchmarks/duckshop/ ## Project shape +The core project, without the [edge-case area](#edge-cases): + - 252 models: 99 staging, 81 intermediate, 66 marts and 6 snapshots. Marts are grouped by area: core, orders, customers, catalogue, inventory, fulfilment, support, clickstream, growth, finance, partners and executive. @@ -50,7 +53,7 @@ benchmarks/duckshop/ ## Tests and scenarios -`tests/unit/` mirrors `models/` and holds 266 unit tests: +`tests/unit/` mirrors `models/` and holds 266 core unit tests: - 260 model tests, including cursor-window tests on incremental models, a table-function mock, a macro mock, a fixture built by a test-only Python macro, an empty fixture for a contracted @@ -58,7 +61,7 @@ benchmarks/duckshop/ - 3 macro tests, 2 UDF tests and 1 table-function test, most driven by `VALUES` case tables; - assertions in 98 tests, including a key check on every mart. -`tests/scenarios/` holds 8 scenarios: one each for catalogue, clickstream, orders, fulfilment, +`tests/scenarios/` holds 8 core scenarios: one each for catalogue, clickstream, orders, fulfilment, support and inventory, and two that cross areas (a regional scorecard, and campaign return on ad spend). Two use `expected_columns "all"`. Expected values are derived by hand from the fixtures, and every test and scenario passes on Snowflake (see [Status](#status)); the scenarios' captured @@ -102,6 +105,43 @@ uv run --isolated --no-project --with pytest --with pyyaml pytest benchmarks/duc See [RUNBOOK.md](RUNBOOK.md) for loading, building and testing on Snowflake. +## Edge cases + +The edge-case area holds small, hand-picked cases for SQL whose results are easy to get subtly +wrong when translating Snowflake SQL or replaying it on another engine. It lives beside the core +project and does not change it: + +- models in `models/staging/edge_cases/`, `models/intermediate/edge_cases/` and + `models/marts/edge_cases/`, each tagged `edge_case`. The `XSQBRDUCK002` Rule fails a model in an + `edge_cases` folder without the tag, or a model with the tag outside one; +- unit tests in the matching `tests/unit//edge_cases/` folders and scenarios in + `tests/scenarios/edge_cases/`; +- raw `edge__*` sources in `sources/edge_cases.yml`, loaded by `python/loaders/edge_cases.py` from + fixed rows that `data_scale` does not change, so core data and core expectations stay the same. + +It has 38 models (7 staging, 22 intermediate, 9 marts), 61 unit tests and 4 scenarios, in four +groups: + +| Group | Covers | +|-------|--------| +| Semi-structured | JSON null against SQL NULL and missing keys; document text with sorted, case-differing and escaped keys; JSON number fidelity and casts out of `VARIANT`; path-like keys (`''`, `*`, `$`-prefixed, dotted) with `GET` and `GET_IGNORE_CASE`; `OBJECT_CONSTRUCT_KEEP_NULL`, `OBJECT_INSERT`, `OBJECT_DELETE`, `OBJECT_PICK`; array builders, `ARRAY_EXCEPT` and number spellings in `ARRAY_POSITION`; `FLATTEN` outer; lenient and malformed `TRY_PARSE_JSON`; typed lambdas | +| Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps | +| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts | +| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | + +Every expected value in the edge-case area was captured on Snowflake under the `edge` target's +pinned session, rather than derived by hand. `fct_edge_delivery_events` is disabled: it is the +acceptance case for cursor `delete_insert` models that emit rows outside the window they replace. +See [RUNBOOK.md](RUNBOOK.md), step 7. + +Benchmarks and core measurements leave the area out: + +- `python benchmarks/duckshop/scripts/core_project.py ` copies the project without + it; the copy compiles to the same files as the project did before the area existed; +- warehouse runs exclude it with `--exclude tag:edge_case "source:edge__*"`, and scenario runs + with `--exclude edge_cases`; +- the construct census counts the core project unless given `--area edge` or `--area all`. + ## Status Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`: @@ -113,7 +153,7 @@ Last verified on Snowflake with SQLBuild 0.175.0 at `data_scale = "1"`: | `sqb test` | 266 of 266 pass | | `sqb scenario test` | 8 of 8 pass | | `sqb scenario capture` | Captures all 8; snapshots are in `project/tests/_scenario_snapshots/` | -| `sqb scenario test --local --strict` | 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot | +| `sqb scenario test --local --strict` | In the core copy, 0 of 8 pass: 5 fail in translation, 3 report a stale snapshot | Local replay translates each scenario's Snowflake SQL to DuckDB on a best-effort basis. These five failures are the baseline that better translation should remove: @@ -132,6 +172,21 @@ The other three scenarios (`campaign_roas__attributes_orders_to_channel_spend`, reported stale straight after capture. The snapshot fingerprint now hashes the authored fixture SQL, so these results predate that fix; recapture the snapshots to replay them. +The project no longer sets `QUOTED_IDENTIFIERS_IGNORE_CASE = true` on its connection. With SQLBuild +0.184.0, a core build without it passes with the same results as with it: every core model holds +the same rows, apart from the snapshots, which record build times, and three models whose results +vary from one run to the next (`int_cart_funnel`, `int_delivery_slot_utilisation` and +`rpt_delivery_exceptions_monthly`). + +The edge-case area, verified on Snowflake with SQLBuild 0.184.0 on the `edge` target: + +| Step | Result | +|------|--------| +| `sqb build --select tag:edge_case --exclude stg_edge_partner_listings+` | Passes; the disabled model is skipped | +| `sqb test --select tag:edge_case` | 61 of 61 pass | +| `sqb scenario test --select edge_cases` | 4 of 4 pass | +| Two-load replay of the delivery change log | The merged status equals the full rebuild after each load | + ## Expected divergences Nine models use constructs whose results depend on the engine or the session. They are tagged @@ -155,6 +210,15 @@ noted here: - Compilation accepts SQL that Snowflake rejects: `CONNECT_BY_ISLEAF`, which Snowflake does not have, and a CTE whose body is a bare `VALUES` list. `int_category_paths` finds leaf categories with a join instead, and test case tables use `SELECT ... FROM VALUES ... AS v(...)`. +- A source column whose name is quoted mixed case, such as `"listingId"`, cannot be read once the + table exists: planning folds the quoted reference to lower case and fails with `B002`. Fixtures + cannot name such columns either (`B302` in unit tests, `S510` in scenarios), and a `MODEL` + header cannot declare a quoted column name. `stg_edge_partner_listings` has a reviewed + `SQBRTEST202` ignore, `rpt_edge_partner_listing_export` a reviewed `SQBRTEST201` ignore, and the + edge build excludes `stg_edge_partner_listings+`. +- Local scenario replay stops at compile (`P001`, unclosed quoted string) on a model with a + backslash-escaped quote in a string literal, `'O\'Brien'` in `stg_edge_ticket_notes`, so local + replay runs in the core copy. - `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the staging models that keep the latest delivery carry reviewed exceptions. diff --git a/benchmarks/duckshop/RUNBOOK.md b/benchmarks/duckshop/RUNBOOK.md index 857f1acce7..4e5605b5a7 100644 --- a/benchmarks/duckshop/RUNBOOK.md +++ b/benchmarks/duckshop/RUNBOOK.md @@ -1,8 +1,12 @@ # DuckShop runbook How to load DuckShop's synthetic data into Snowflake, build and test the project, capture the -scenario snapshots, and clean up afterwards. Run every command from `benchmarks/duckshop/project` -unless it says otherwise. +scenario snapshots, run the edge-case area, and clean up afterwards. Run every command from +`benchmarks/duckshop/project` unless it says otherwise. + +Steps 3 to 6 run the core project only: they exclude the edge-case area (models tagged +`edge_case`, their `edge__*` sources and the `edge_cases` scenario folder), which step 7 runs on its +own target. ## 1. Configure the connection @@ -15,11 +19,13 @@ cp sqlbuild_local.toml.example sqlbuild_local.toml ``` Fill in the account, user, authenticator, role and warehouse, and the `database` and `schema` of -`[targets.dev]`. Keep the project's `session_parameters` setting on the connection (see Known -limitations in the README): `sqlbuild_local.toml` adds to the `[connections.warehouse]` table in -`sqlbuild_project.toml` rather than replacing it, so leave `session_parameters` out of the local -file. `sqlbuild_local.toml` is gitignored; never commit it, and never copy its values -into tracked files, commit messages or pull requests. +`[targets.dev]`. For the edge-case area (step 7), fill in `[connections.edge]` and `[targets.edge]` +the same way, with a different schema from `[targets.dev]`: two targets that write managed sources +to the same schema fail with `S102`. `sqlbuild_local.toml` adds to the connection tables in +`sqlbuild_project.toml` rather than replacing them, so the edge connection keeps the project's +pinned `session_parameters`; leave `session_parameters` out of the local file. +`sqlbuild_local.toml` is gitignored; never commit it, and never copy its values into tracked +files, commit messages or pull requests. Check the connection, then confirm the project compiles: @@ -43,13 +49,15 @@ sqb query --file target/check.sql --no-limit ``` The query lists every existing table, view or function with a DuckShop name, which is everything -step 7 drops, and any SQLBuild state tables (`_SQLBUILD_*`). Before the first build it should list -no DuckShop names; if it does, stop and pick another schema. Note which state tables already existed, for the cleanup in step 7. +step 8 drops, and any SQLBuild state tables (`_SQLBUILD_*`). Before the first build it should list +no DuckShop names; if it does, stop and pick another schema. Note which state tables already existed, for the cleanup in step 8. +The list includes the edge-case area's names; run the check with `--target edge` too before its +first build. ## 3. Load the synthetic data ```bash -sqb load --concurrency 4 +sqb load --concurrency 4 --exclude "source:edge__*" ``` All 76 managed sources load, about 60,000 rows in total. Each loader replaces its table, so loading @@ -64,7 +72,7 @@ data_scale = "10" ## 4. Build ```bash -sqb build --concurrency 8 +sqb build --concurrency 8 --exclude tag:edge_case "source:edge__*" ``` The build reloads the managed sources first. SQLBuild 0.175.0 does this even with `--no-load` @@ -108,8 +116,8 @@ sqb build --select fct_web_sessions --full-refresh \ ## 5. Run the unit tests and scenarios on Snowflake ```bash -sqb test --concurrency 8 -sqb scenario test --concurrency 4 +sqb test --concurrency 8 --exclude tag:edge_case +sqb scenario test --concurrency 4 --exclude edge_cases ``` Unit tests and scenarios read only their own fixtures, not the loaded data, so they can run before @@ -119,14 +127,17 @@ on Snowflake. Report any failure rather than editing the expectations to match. ## 6. Capture the scenario snapshots and commit them ```bash -sqb scenario capture -sqb scenario test --local --strict +sqb scenario capture --exclude edge_cases +python ../scripts/core_project.py /tmp/duckshop-core +(cd /tmp/duckshop-core && sqb scenario test --local --strict) ``` Capture materialises each scenario's fixtures in Snowflake, downloads them as JSONL into `tests/_scenario_snapshots/`, and drops what it created. The local run then replays all 8 scenarios on your machine from those files, with no warehouse connection; `--strict` fails on a -missing or stale snapshot instead of skipping it. +missing or stale snapshot instead of skipping it. It runs in a copy of the project without the +edge-case area, because local replay of the full project stops at compile on an edge-case model +(see Known limitations in the README). Delete the copy afterwards. With SQLBuild 0.175.0 the local run is expected to end with `ERROR=8`: five scenarios fail while translating Snowflake SQL to DuckDB, and three report `X602` (stale snapshot) because of a @@ -144,7 +155,80 @@ git commit -m "test(benchmarks): capture DuckShop scenario snapshots" The `grep` must print nothing. The repository's publication checks run on commit and push as usual. -## 7. Clean up +## 7. Edge cases + +The edge-case area runs on the `edge` target, whose connection pins the session its expected +values were captured under: UTC, and Snowflake's defaults for every parameter that changes a +result or how text is parsed, including `QUOTED_IDENTIFIERS_IGNORE_CASE = false`. Run it on that +target only; under another session some expected values differ by design. + +### Load, build and test + +```bash +sqb build --target edge --select tag:edge_case --exclude stg_edge_partner_listings+ --concurrency 8 +sqb test --target edge --select tag:edge_case --concurrency 8 +sqb scenario test --target edge --select edge_cases +``` + +The build loads the `edge__*` sources from fixed rows that `data_scale` does not change, then +builds the edge models and runs their tests and audits. `fct_edge_delivery_events` is disabled and +reports `skip: disabled` (see [Pending acceptance cases](#pending-acceptance-cases)). +`stg_edge_partner_listings` and its downstream models are excluded because planning fails once the +`edge__partner_listings` table exists (see Known limitations in the README); the +`edge_partner_listings__star_contract_and_quoted_export` scenario covers the models downstream of +it. + +### Replay the delivery change log as two loads + +`int_edge_delivery_status` merges a shipment delivery change log incrementally, and the +`edge_delivery_status_matches_rebuild` audit fails if it differs from a rebuild from the whole log. +To exercise the merge across loads, start without the table, load the first load only, build, then +load everything and build again: + +```bash +sqb query --target edge "DROP TABLE IF EXISTS ..int_edge_delivery_status" +sqb load --target edge --select edge__delivery_updates --vars '{"edge_delivery_batch": "1"}' +sqb build --target edge --no-load \ + --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt +sqb load --target edge --select edge__delivery_updates +sqb build --target edge --no-load \ + --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt +``` + +The second build plans `int_edge_delivery_status` as a `merge` over the second load, which carries +a late older version, a redelivered version, a delete and a new shipment, and the audit passes. +Load with `sqb load` and build with `--no-load`, so that each build reads exactly the load you +chose. + +### Capture expected values + +Edge-case expectations are Snowflake's own results on the fixtures, not values derived by hand. +To capture them for a new or changed edge test, write its `__expected__` CTE with the +columns to compare (any values, such as `NULL AS column_name`), then run: + +```bash +python ../scripts/capture_edge_expectations.py --write ... +sqb format tests/unit/staging/edge_cases tests/unit/intermediate/edge_cases tests/unit/marts/edge_cases +``` + +The script runs the edge tests on the `edge` target, reads each selected test's executed SQL, and +replaces each expected CTE with the actual rows as typed literals. Without a test name it +recaptures every edge test, which must leave the files unchanged. Check every captured value +against the case the test describes before committing: a capture records what Snowflake returned, +which is the point, but a fixture mistake is captured as faithfully as a real edge. Scenario +expectations are written by hand from the captured unit-test values. + +Capture scenario snapshots for the edge scenarios with +`sqb scenario capture --target edge --select edge_cases`. + +### Pending acceptance cases + +`fct_edge_delivery_events` filters its input by order date but is rebuilt by delivery date, so a +rerun of a window deletes only that window's deliveries and inserts deliveries outside it again. +It stays disabled (`enabled false`) until cursor `delete_insert` models guard rows outside the +window; its unit test records the rows outside the window that the model emits today. + +## 8. Clean up Drop everything the project created, by name: @@ -156,7 +240,9 @@ sqb clean ``` `drop.sql` is one Snowflake Scripting block of `DROP ... IF EXISTS` statements for every model, -snapshot, seed, raw source table and function, so it is safe to run twice. Scenario runs and +snapshot, seed, raw source table and function, edge-case area included, so it is safe to run +twice. Run the same three commands with `--target edge` (on `sqb compile` and `sqb query`) to clean +up the edge target. Scenario runs and captures already drop their own relations. SQLBuild also keeps state tables (`_SQLBUILD_*`) in the schema. If step 2 showed none before the diff --git a/benchmarks/duckshop/project/python/loaders/_edge_cases.py b/benchmarks/duckshop/project/python/loaders/_edge_cases.py index 310fef3ca4..b555839004 100644 --- a/benchmarks/duckshop/project/python/loaders/_edge_cases.py +++ b/benchmarks/duckshop/project/python/loaders/_edge_cases.py @@ -25,9 +25,11 @@ def partner_payloads() -> Rows: '{"sku": "MALLARD-01", "qty": 2, "amount": 19.990000000000002, ' '"txn_id": 12345678901234567890, "gift": "yes", "placed_on": "2026-03-01", ' '"colour": null, "channel": "web", "currency": "GBP", ' - '"items": [{"sku": "MALLARD-01", "qty": 2, "price": 4.995}, {"QTY": 2, "sku": "TEAL-02"}], ' + '"items": [{"sku": "MALLARD-01", "qty": 2, "price": 4.995}, ' + '{"QTY": 2, "sku": "TEAL-02"}], ' '"attrs": {"Size (cm)": "12", "a.b": "dotted", "a": {"b": "nested"}, ' - '"$discount": "5%", "": "blank", "*": "star", "$.a": "dollar path", "Sku": "MALLARD-01"}, ' + '"$discount": "5%", "": "blank", "*": "star", "$.a": "dollar path", ' + '"Sku": "MALLARD-01"}, ' '"meta": {"order_id": "DS-O-1001"}, "tags": ["bath", "gift", "bath", null], ' '"removed_tags": ["bath"], "extra_tags": ["gift"], "promo_code": "SPRING", ' '"gift_code": null, "category_id": 2, "eligible_ids": [1, 2.0, 3], ' @@ -177,22 +179,80 @@ def price_windows() -> Rows: def customer_profiles() -> Rows: profiles = [ - # customer, full name, previous name, first, middle, last, line 1, line 2, tier, number, updated - ("C-1", "Größe Ente", "Grosse Ente", "Jürgen", None, "Größe", "1 Pond Lane", None, - "Golden Duck", 12345678901234567890, "2026-03-01 09:00:00"), - ("C-1", "Größe Ente", "Grosse Ente", "Jürgen", None, "Größe", "1 Pond Lane", "Flat 2", - "Golden Duck", 12345678901234567890, None), - ("C-2", "e\u0301lodie o'neill-smith", "elodie oneill smith", "Élodie", "", "O'Neill", - "2 Marsh Road", "", "Duckling", 9007199254740993, "2026-03-02 10:00:00"), - ("C-3", "ijsselmeer van der eend", "ijsselmeer van der eend", "IJsbrand", "van der", "Eend", - None, None, "BLOCKED", 9007199254740992, "2026-03-03 11:00:00"), + # customer, full name, previous name, first, middle, last, address lines 1 and 2, tier, + # partner order number, updated at + ( + "C-1", + "Größe Ente", + "Grosse Ente", + "Jürgen", + None, + "Größe", + "1 Pond Lane", + None, + "Golden Duck", + 12345678901234567890, + "2026-03-01 09:00:00", + ), + ( + "C-1", + "Größe Ente", + "Grosse Ente", + "Jürgen", + None, + "Größe", + "1 Pond Lane", + "Flat 2", + "Golden Duck", + 12345678901234567890, + None, + ), + ( + "C-2", + "e\u0301lodie o'neill-smith", + "elodie oneill smith", + "Élodie", + "", + "O'Neill", + "2 Marsh Road", + "", + "Duckling", + 9007199254740993, + "2026-03-02 10:00:00", + ), + ( + "C-3", + "ijsselmeer van der eend", + "ijsselmeer van der eend", + "IJsbrand", + "van der", + "Eend", + None, + None, + "BLOCKED", + 9007199254740992, + "2026-03-03 11:00:00", + ), (None, "ann (dup)", None, None, None, None, None, None, "BLOCKED", None, None), - ("C-4", "ნინო ბერიძე", "ნინო ბერიძე", "ნინო", None, "ბერიძე", "4 Nest Street", - "Unit 🦆", "Mallard", -9223372036854775809, "2026-03-04 12:00:00"), + ( + "C-4", + "ნინო ბერიძე", + "ნინო ბერიძე", + "ნინო", + None, + "ბერიძე", + "4 Nest Street", + "Unit 🦆", + "Mallard", + -9223372036854775809, + "2026-03-04 12:00:00", + ), ] rows: Rows = [] for index, profile in enumerate(profiles): - (customer, full, previous, first, middle, last, line1, line2, tier, number, updated) = profile + (customer, full, previous, first, middle, last, line1, line2, tier, number, updated) = ( + profile + ) rows.append( { "profile_id": f"PR-{index + 1:04d}", @@ -217,26 +277,145 @@ def ledger_entries() -> Rows: entries = [ # order, channel, line, sku, warehouse, qty, qty text, price, amount, currency, rate, # status, reason, tier, changed at - ("DS-O-2001", "web", 1, "MALLARD-01", "POND", 3, "007", 1299, "2.5", "GBP", 1.27, - "shipped", None, "Golden Duck", "2026-03-01 09:00:00"), - ("DS-O-2001", "web", 1, "TEAL-02", "POND", 1, "+7", 1299, "-2.5", "GBP", 1.27, - "Shipped", "REFUND", "Golden Duck", "2026-03-01 09:00:00"), - ("DS-O-2001", "web", 2, "EIDER-03", "POND", 2, "1e0", 899, "0.005", "GBP", 0.1, - "allocated", None, "Golden Duck", None), - ("DS-O-2002", "app", 1, "MALLARD-01", "NEST", 0, " 1.5 ", 1350, "1.0000000005", "KWD", - 3.25, "counted", "DAMAGED", "Duckling", "2026-03-02 09:00:00"), - ("DS-O-2002", "app", 2, "PINTAIL-04", "NEST", -4, "-0", None, "-7.5", "JPY", 0.0067, - "ALLOCATED", None, "Duckling", "2026-03-02 09:00:00"), - ("DS-O-2003", "partner", 1, "TEAL-02", "MARSH", 5, "1,000", 1100, - "123456789012345678901234567.0123456789", "AUD", 0.66, "Counted", None, None, - "2026-03-03 09:00:00"), - ("DS-O-2003", "partner", 2, "TEAL-02", "MARSH", 5, "n/a", 1100, "0", "AUD", 0.66, - "pending", None, None, "2026-03-03 09:00:00"), + ( + "DS-O-2001", + "web", + 1, + "MALLARD-01", + "POND", + 3, + "007", + 1299, + "2.5", + "GBP", + 1.27, + "shipped", + None, + "Golden Duck", + "2026-03-01 09:00:00", + ), + ( + "DS-O-2001", + "web", + 1, + "TEAL-02", + "POND", + 1, + "+7", + 1299, + "-2.5", + "GBP", + 1.27, + "Shipped", + "REFUND", + "Golden Duck", + "2026-03-01 09:00:00", + ), + ( + "DS-O-2001", + "web", + 2, + "EIDER-03", + "POND", + 2, + "1e0", + 899, + "0.005", + "GBP", + 0.1, + "allocated", + None, + "Golden Duck", + None, + ), + ( + "DS-O-2002", + "app", + 1, + "MALLARD-01", + "NEST", + 0, + " 1.5 ", + 1350, + "1.0000000005", + "KWD", + 3.25, + "counted", + "DAMAGED", + "Duckling", + "2026-03-02 09:00:00", + ), + ( + "DS-O-2002", + "app", + 2, + "PINTAIL-04", + "NEST", + -4, + "-0", + None, + "-7.5", + "JPY", + 0.0067, + "ALLOCATED", + None, + "Duckling", + "2026-03-02 09:00:00", + ), + ( + "DS-O-2003", + "partner", + 1, + "TEAL-02", + "MARSH", + 5, + "1,000", + 1100, + "123456789012345678901234567.0123456789", + "AUD", + 0.66, + "Counted", + None, + None, + "2026-03-03 09:00:00", + ), + ( + "DS-O-2003", + "partner", + 2, + "TEAL-02", + "MARSH", + 5, + "n/a", + 1100, + "0", + "AUD", + 0.66, + "pending", + None, + None, + "2026-03-03 09:00:00", + ), ] rows: Rows = [] for index, entry in enumerate(entries): - (order, channel, line, sku, warehouse, qty, qty_text, price, amount, currency, rate, - status, reason, tier, changed) = entry + ( + order, + channel, + line, + sku, + warehouse, + qty, + qty_text, + price, + amount, + currency, + rate, + status, + reason, + tier, + changed, + ) = entry rows.append( { "entry_id": f"LE-{index + 1:04d}", @@ -263,10 +442,20 @@ def ledger_entries() -> Rows: def ticket_notes() -> Rows: notes = [ - ("Customer O'Brien asks about DS-O-1001 and DS-O-1002; refund\tplease", - "bath#gift#teal", "newsletter", "50%OFF", "MALLARD-01"), - ("Order ds-o-77 was REFUNDED.\nPath C:\\ducks\\pond # not a comment", - "#leading", "Email", "STAFF10", "DUCK--"), + ( + "Customer O'Brien asks about DS-O-1001 and DS-O-1002; refund\tplease", + "bath#gift#teal", + "newsletter", + "50%OFF", + "MALLARD-01", + ), + ( + "Order ds-o-77 was REFUNDED.\nPath C:\\ducks\\pond # not a comment", + "#leading", + "Email", + "STAFF10", + "DUCK--", + ), ("No order here", "", "partner_feed", "50PERCENT", ""), (None, None, None, None, "A-B-C-D"), ] @@ -338,7 +527,9 @@ def delivery_updates(batch: int = 2) -> Rows: "_seq": seq, "_loaded_at": _at(loaded), } - for index, (loaded, shipment, order, status, ordered, delivered, op, seq) in enumerate(updates) + for index, (loaded, shipment, order, status, ordered, delivered, op, seq) in enumerate( + updates + ) if loaded in loads ] diff --git a/benchmarks/duckshop/project/sqlbuild_local.toml.example b/benchmarks/duckshop/project/sqlbuild_local.toml.example index 16d18f6db8..468e2d2702 100644 --- a/benchmarks/duckshop/project/sqlbuild_local.toml.example +++ b/benchmarks/duckshop/project/sqlbuild_local.toml.example @@ -13,3 +13,16 @@ warehouse = "" connection = "warehouse" database = "" schema = "" + +# The edge-case area's target (RUNBOOK.md, step 7). Use a different schema from targets.dev. +[connections.edge] +account = "" +user = "" +authenticator = "" +role = "" +warehouse = "" + +[targets.edge] +connection = "edge" +database = "" +schema = "" diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql index 5660a2b3b7..71e580ae2e 100644 --- a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_partner_listing_export.sql @@ -13,11 +13,17 @@ WITH __ref__dim_edge_partner_listings AS ( __expected__rpt_edge_partner_listing_export AS ( SELECT - v.listing_id AS "listingId", - v.unit_price AS "unitPrice", - v.is_live AS "isLive" - FROM VALUES - ('LS-MALLARD-01', 12.99, TRUE), - ('LS-TEAL-02', 8.99, FALSE), - ('LS-EIDER-03', 0.05, NULL) AS v(listing_id, unit_price, is_live) + 'LS-EIDER-03' AS "listingId", + 0.05::NUMBER(18, 2) AS "unitPrice", + CAST(NULL AS BOOLEAN) AS "isLive" + UNION ALL + SELECT + 'LS-MALLARD-01' AS "listingId", + 12.99::NUMBER(18, 2) AS "unitPrice", + TRUE AS "isLive" + UNION ALL + SELECT + 'LS-TEAL-02' AS "listingId", + 8.99::NUMBER(18, 2) AS "unitPrice", + FALSE AS "isLive" ) diff --git a/benchmarks/duckshop/scripts/capture_edge_expectations.py b/benchmarks/duckshop/scripts/capture_edge_expectations.py new file mode 100644 index 0000000000..bc275c9189 --- /dev/null +++ b/benchmarks/duckshop/scripts/capture_edge_expectations.py @@ -0,0 +1,278 @@ +"""Capture the expected rows of DuckShop's edge-case unit tests from Snowflake. + +Run it from `benchmarks/duckshop/project` with the `edge` target configured (see RUNBOOK.md, +step 7): + + python ../scripts/capture_edge_expectations.py [--write] [TEST_NAME ...] + +It compiles the project and runs the edge-case unit tests on the `edge` target, which writes each +test's executed SQL to `target/run/tests/`; the newest file per test is read. For every +`__expected__` CTE of the selected tests (all edge-case tests by default), it then runs the +model under test on its fixtures, reads the actual rows of the columns the CTE lists, and renders +them as typed SQL literals in a stable order. Without `--write` it prints them; with `--write` it +replaces the CTE body in the test file. Review every captured value before committing it: a +capture records what Snowflake returned, right or wrong. + +The script reads the executed test SQL that SQLBuild writes, so a change to that SQL's shape may +need a change here. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import re +import subprocess +import sys +import unicodedata + +SQB = os.environ.get("SQB", "sqb") +TARGET = "edge" +LITERAL_WIDTH = 90 +TEST_HEADER = re.compile(r'^TEST \(\s*name "([^"]+)"', re.MULTILINE) +EXPECTED_COLUMNS = re.compile(r"\(SELECT ((?:(?!SELECT).)*?) FROM __actual__(\w+) EXCEPT") +STRING_LITERAL = re.compile(r"'(?:[^'\\]|\\.|'')*'") +ESCAPED_CATEGORIES = ("Cc", "Cf", "Mn", "Zl", "Zp") + + +def run_sqb(project: pathlib.Path, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [SQB, "--no-color", *arguments], + cwd=project, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + check=False, + ) + + +def query(project: pathlib.Path, sql: str) -> list[dict[str, object]]: + path = project / "target" / "capture_query.sql" + path.write_text(sql) + result = run_sqb( + project, "query", "--target", TARGET, "--file", str(path), "--json", "--no-limit" + ) + documents = [line for line in result.stdout.splitlines() if line.startswith("[")] + if result.returncode != 0 or not documents: + raise RuntimeError(f"query failed:\n{result.stdout[-2000:]}\n{result.stderr[-2000:]}") + return json.loads(documents[-1]) + + +def test_blocks(text: str) -> list[tuple[str, int, int]]: + """(name, start, end) of every TEST block in a test file.""" + + starts = list(TEST_HEADER.finditer(text)) + return [ + (match.group(1), match.start(), starts[i + 1].start() if i + 1 < len(starts) else len(text)) + for i, match in enumerate(starts) + ] + + +def executed_test_sql(project: pathlib.Path, name: str) -> tuple[str, str]: + """The fixture and model CTEs of an executed test, and its final comparison SELECT.""" + + hits = list((project / "target" / "run" / "tests").rglob(f"{name}.sql")) + if not hits: + raise RuntimeError(f"no executed SQL for {name}") + sql = max(hits, key=lambda hit: hit.stat().st_mtime).read_text() + comparison = re.search(r"\nSELECT \d+ AS step_index", sql) + if comparison is None: + raise RuntimeError(f"no comparison SELECT in the executed SQL of {name}") + return sql[: comparison.start()], sql[comparison.start() :] + + +def render_expression(column: str, snowflake_type: str) -> str: + """Select a column as text that round-trips exactly through a typed literal.""" + + base = snowflake_type.split("[")[0].split("(")[0] + formats = { + "DATE": "YYYY-MM-DD", + "TIME": "HH24:MI:SS.FF9", + "TIMESTAMP_NTZ": "YYYY-MM-DD HH24:MI:SS.FF9", + "TIMESTAMP_TZ": "YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM", + "TIMESTAMP_LTZ": "YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM", + } + if base == "FLOAT": + return column + if base == "NUMBER": + return f"TO_VARCHAR({column})" + if base in formats: + return f"TO_VARCHAR({column}, '{formats[base]}')" + if base in ("VARIANT", "OBJECT", "ARRAY"): + return f"TO_JSON({column})" + if base == "BINARY": + return f"HEX_ENCODE({column})" + return f"{column}::VARCHAR" + + +def quote(text: str) -> str: + """A Snowflake string literal, escaping characters that are invisible or easily confused.""" + + out = [] + for char in text: + if char == "\\": + out.append("\\\\") + elif char == "'": + out.append("''") + elif ord(char) <= 0xFFFF and unicodedata.category(char) in ESCAPED_CATEGORIES: + out.append(f"\\u{ord(char):04x}") + else: + out.append(char) + return "'" + "".join(out) + "'" + + +def literal(value: object, snowflake_type: str) -> str: + sql_type = snowflake_type.split("[")[0] + if sql_type.startswith("VARCHAR"): + sql_type = "VARCHAR" + base = sql_type.split("(")[0] + if value is None: + return "NULL" if base == "NULL" else f"CAST(NULL AS {sql_type})" + if base == "FLOAT": + return f"'{value if isinstance(value, str) else repr(float(value))}'::FLOAT" # type: ignore[arg-type] + if base == "NUMBER": + scale = re.match(r"NUMBER\(\d+,(\d+)\)", sql_type) + return f"{value}::{sql_type}" if scale and scale.group(1) != "0" else str(value) + if base == "BOOLEAN": + return "TRUE" if str(value).lower() == "true" else "FALSE" + if base == "DATE": + return f"'{value}'::DATE" + if base in ("TIME", "TIMESTAMP_NTZ", "TIMESTAMP_TZ", "TIMESTAMP_LTZ"): + text = re.sub( + r"\.(\d*?)0+(?=$| )", lambda m: f".{m.group(1)}" if m.group(1) else "", str(value) + ) + return f"'{text}'::{sql_type}" + if base == "VARIANT": + return f"PARSE_JSON({quote(str(value))})" + if base in ("OBJECT", "ARRAY"): + return f"PARSE_JSON({quote(str(value))})::{base}" + if base == "BINARY": + return f"HEX_DECODE_BINARY('{value}')" + return quote(str(value)) + + +def capture(project: pathlib.Path, ctes: str, model: str, columns: list[str]) -> str | None: + """The actual rows of a model's listed columns, as the body of its expected CTE.""" + + probe = ", ".join(f"SYSTEM$TYPEOF({column}) AS t{i}" for i, column in enumerate(columns)) + typed = query(project, f"{ctes}\nSELECT {probe} FROM __actual__{model} LIMIT 1") + if not typed: + return None + types = [str(typed[0][f"T{i}"]) for i in range(len(columns))] + rendered = ", ".join( + f"{render_expression(column, kind)} AS v{i}" + for i, (column, kind) in enumerate(zip(columns, types, strict=True)) + ) + rows = query(project, f"{ctes}\nSELECT {rendered} FROM __actual__{model}") + values = sorted([literal(row[f"V{i}"], kind) for i, kind in enumerate(types)] for row in rows) + selects = [ + "SELECT\n" + ",\n".join(f" {v} AS {c}" for v, c in zip(row, columns, strict=True)) + for row in values + ] + return "\n UNION ALL\n ".join(selects) + + +def replace_expected(text: str, start: int, end: int, model: str, body: str) -> str: + match = re.compile(rf"__expected__{model} AS \(\n").search(text, start, end) + if match is None: + raise RuntimeError(f"no __expected__{model} CTE") + depth, index, in_string = 1, match.end(), False + while depth: + char = text[index] + if in_string: + if char == "\\": + index += 1 + elif char == "'": + in_string = False + elif char == "'": + in_string = True + elif char in "()": + depth += 1 if char == "(" else -1 + index += 1 + return text[: match.end()] + " " + body + "\n" + text[index - 1 :] + + +def wrap_literal(text: str, indent: str) -> str: + """Split a long string literal into `||` parts, never inside an escape or doubled quote.""" + + body, parts = text[1:-1], [] + while len(body) > LITERAL_WIDTH: + cut = body.rfind(", ", 0, LITERAL_WIDTH) + cut = cut + 2 if cut > 0 else body.rfind(",", 0, LITERAL_WIDTH) + 1 or LITERAL_WIDTH + escape = body.rfind("\\", max(0, cut - 6), cut) + if escape >= 0: + while escape > 0 and body[escape - 1] == "\\": + escape -= 1 + cut = escape + while body[cut - 1] == "'" and body[cut : cut + 1] == "'": + cut -= 1 + parts.append(body[:cut]) + body = body[cut:] + parts.append(body) + return f"\n{indent}|| ".join(f"'{part}'" for part in parts) + + +def wrap_line(line: str) -> str: + indent = line[: len(line) - len(line.lstrip())] + return STRING_LITERAL.sub( + lambda m: ( + wrap_literal(m.group(0), indent) if len(m.group(0)) > LITERAL_WIDTH + 10 else m.group(0) + ), + line, + ) + + +def wrap_long_literals(text: str) -> str: + return "".join(wrap_line(line) for line in text.splitlines(keepends=True)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("tests", nargs="*", help="test names to capture (default: all edge tests)") + parser.add_argument("--write", action="store_true", help="write the rows into the test files") + args = parser.parse_args() + project = pathlib.Path.cwd() + + compiled = run_sqb(project, "compile") + if " 0 errors" not in compiled.stdout + compiled.stderr: + print((compiled.stdout + compiled.stderr)[-3000:], file=sys.stderr) + return 1 + tested = run_sqb(project, "test", "--target", TARGET, "--select", "tag:edge_case") + if "TOTAL=" not in tested.stdout + tested.stderr: + print((tested.stdout + tested.stderr)[-3000:], file=sys.stderr) + return 1 + + failures = 0 + for path in sorted((project / "tests" / "unit").glob("*/edge_cases/*.sql")): + for name, _, _ in test_blocks(path.read_text()): + if args.tests and name not in args.tests: + continue + try: + ctes, comparison = executed_test_sql(project, name) + except RuntimeError as error: + print(f"!! {name}: {error}", file=sys.stderr) + failures += 1 + continue + expected = {m.group(2): m.group(1) for m in EXPECTED_COLUMNS.finditer(comparison)} + for model, listed in expected.items(): + columns = [column.strip() for column in listed.split(",")] + try: + body = capture(project, ctes, model, columns) + except RuntimeError as error: + print(f"!! {name} {model}: {error}", file=sys.stderr) + failures += 1 + continue + print(f"== {name} {model}\n {body or '(no rows)'}\n") + if args.write and body is not None: + text = path.read_text() + _, start, end = next(b for b in test_blocks(text) if b[0] == name) + path.write_text( + wrap_long_literals(replace_expected(text, start, end, model, body)) + ) + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/duckshop/scripts/census.py b/benchmarks/duckshop/scripts/census.py index 80e75f5214..fc431e6f78 100644 --- a/benchmarks/duckshop/scripts/census.py +++ b/benchmarks/duckshop/scripts/census.py @@ -2,11 +2,12 @@ Run `sqb compile` in ../project first, then: - python benchmarks/duckshop/scripts/census.py [--json] + python benchmarks/duckshop/scripts/census.py [--json] [--area core|edge|all] Constructs are detected with regular expressions over the compiled model SQL, with comments and string literals removed. A model counts towards a family when the family appears at least once. -The output is aggregate only. +The output is aggregate only. By default it counts the core project only: models under an +`edge_cases` folder belong to the edge-case area, which `--area edge` or `--area all` selects. """ from __future__ import annotations @@ -20,6 +21,8 @@ PROJECT = pathlib.Path(__file__).resolve().parent.parent / "project" COMPILED = PROJECT / "target" / "compiled" / "models" +EDGE_FOLDER = "edge_cases" +AREAS = ("core", "edge", "all") STRING = re.compile(r"'(?:[^'\\]|\\.|'')*'", re.S) TOKENS = re.compile(r"('(?:[^'\\]|\\.|'')*')|(/\*.*?\*/)|((?:--|//)[^\n]*)", re.S) @@ -212,11 +215,22 @@ def layer_of(path: pathlib.Path) -> str: return path.relative_to(COMPILED).parts[0] +def in_area(path: pathlib.Path, area: str) -> bool: + is_edge = EDGE_FOLDER in path.relative_to(COMPILED).parts + return area == "all" or is_edge == (area == "edge") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--json", action="store_true", help="print JSON instead of a table") + parser.add_argument( + "--area", + choices=AREAS, + default="core", + help="core models only (the default), the edge-case area only, or both", + ) args = parser.parse_args() - files = sorted(COMPILED.rglob("*.sql")) + files = sorted(path for path in COMPILED.rglob("*.sql") if in_area(path, args.area)) if not files: print(f"no compiled models under {COMPILED}; run `sqb compile` first", file=sys.stderr) return 1 diff --git a/benchmarks/duckshop/scripts/core_project.py b/benchmarks/duckshop/scripts/core_project.py new file mode 100644 index 0000000000..68bd4d8438 --- /dev/null +++ b/benchmarks/duckshop/scripts/core_project.py @@ -0,0 +1,43 @@ +"""Copy the DuckShop project without its edge-case area, for benchmarks and core measurements. + + python benchmarks/duckshop/scripts/core_project.py + +The edge-case area holds hand-picked edge values, not benchmark workload, so compile benchmarks +measure this copy. It leaves out every file and folder whose name starts with `edge_` or `_edge_`: +the `edge_cases` model, test and scenario folders, their source file and loaders, and their +scenario snapshots. Local-only files such as `target/` and `sqlbuild_local.toml` are left out too. +""" + +from __future__ import annotations + +import argparse +import pathlib +import shutil +import sys + +PROJECT = pathlib.Path(__file__).resolve().parent.parent / "project" +EDGE_AREA_PATTERNS = ("edge_*", "_edge_*") +LOCAL_ONLY_PATTERNS = ("target", "logs", ".sqlbuild", "sqlbuild_local.toml", "__pycache__") + + +def copy_core_project(destination: pathlib.Path) -> None: + shutil.copytree( + PROJECT, + destination, + ignore=shutil.ignore_patterns(*EDGE_AREA_PATTERNS, *LOCAL_ONLY_PATTERNS), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("destination", type=pathlib.Path, help="new folder for the copy") + args = parser.parse_args() + if args.destination.exists(): + print(f"{args.destination} already exists", file=sys.stderr) + return 1 + copy_core_project(args.destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/duckshop/tests/test_synthetic_data.py b/benchmarks/duckshop/tests/test_synthetic_data.py index ac2ba9ece8..1d8ff54725 100644 --- a/benchmarks/duckshop/tests/test_synthetic_data.py +++ b/benchmarks/duckshop/tests/test_synthetic_data.py @@ -22,22 +22,29 @@ PROJECT_DIR = Path(__file__).resolve().parents[1] / "project" sys.path.insert(0, str(PROJECT_DIR)) -from python.loaders import _world # noqa: E402 +from python.loaders import _edge_cases, _world # noqa: E402 from python.loaders._registry import GENERATORS, generate, scale_from_vars # noqa: E402 Rows = list[dict[str, object]] -def _sources() -> dict[str, dict[str, object]]: +EDGE_SOURCES_FILE = PROJECT_DIR / "sources" / "edge_cases.yml" + + +def _sources(*, edge: bool) -> dict[str, dict[str, object]]: sources: dict[str, dict[str, object]] = {} for path in sorted((PROJECT_DIR / "sources").glob("*.yml")): + if (path == EDGE_SOURCES_FILE) != edge: + continue for source in yaml.safe_load(path.read_text())["sources"]: sources[source["name"]] = source return sources -SOURCES = _sources() +SOURCES = _sources(edge=False) MANAGED = sorted(name for name, source in SOURCES.items() if source.get("managed")) +EDGE_SOURCES = _sources(edge=True) +EDGE_MANAGED = sorted(name for name, source in EDGE_SOURCES.items() if source.get("managed")) @pytest.fixture(scope="module") @@ -426,3 +433,68 @@ def test_given_special_characters_when_rendering_literals_then_they_are_escaped( assert literal(None) == "NULL" assert literal(True) == "TRUE" assert literal(date(2026, 3, 31)) == "'2026-03-31'" + + +def test_given_edge_sources_when_listing_fixed_tables_then_each_has_rows_and_a_loader() -> None: + loaders = (PROJECT_DIR / "python" / "loaders" / "edge_cases.py").read_text() + + assert EDGE_MANAGED + assert sorted(_edge_cases.TABLES) == EDGE_MANAGED + assert not set(EDGE_MANAGED) & set(GENERATORS) + for name in EDGE_MANAGED: + assert f"def {name}(ctx: LoaderContext)" in loaders, name + + +@pytest.mark.parametrize("name", EDGE_MANAGED) +def test_given_edge_rows_when_compared_to_source_yaml_then_shape_matches(name: str) -> None: + rows = _edge_cases.TABLES[name] + columns = EDGE_SOURCES[name]["columns"] + declared = [column["name"] for column in columns] # type: ignore[attr-defined] + + assert rows, f"{name} has no rows" + for row in rows: + assert list(row) == declared, f"{name}: {sorted(set(row) ^ set(declared))}" + for column in declared: + kinds = {_kind(row[column]) for row in rows} - {None} + assert len(kinds) <= 1, f"{name}.{column} mixes {kinds}" + for column in columns: # type: ignore[attr-defined] + values = [row[column["name"]] for row in rows] + if column.get("nullable") is False: + assert None not in values, f"{name}.{column['name']} has NULLs" + if "unique" in (column.get("audits") or []): + assert len(set(values)) == len(values), f"{name}.{column['name']} repeats" + + +@pytest.mark.parametrize("scale", ["1", "10"]) +def test_given_any_data_scale_when_building_edge_rows_then_they_are_fixed(scale: str) -> None: + assert scale_from_vars({"data_scale": scale}) == int(scale) + rebuilt = { + "edge__partner_payloads": _edge_cases.partner_payloads(), + "edge__gateway_callbacks": _edge_cases.gateway_callbacks(), + "edge__price_windows": _edge_cases.price_windows(), + "edge__customer_profiles": _edge_cases.customer_profiles(), + "edge__ledger_entries": _edge_cases.ledger_entries(), + "edge__ticket_notes": _edge_cases.ticket_notes(), + "edge__partner_listings": _edge_cases.partner_listings(), + "edge__delivery_updates": _edge_cases.delivery_updates(), + } + + assert {name: _digest(rows) for name, rows in rebuilt.items()} == { + name: _digest(rows) for name, rows in _edge_cases.TABLES.items() + } + + +def test_given_first_delivery_batch_when_loading_then_it_is_the_first_load_of_the_log() -> None: + first = _edge_cases.delivery_updates(1) + full = _edge_cases.delivery_updates(2) + first_load = min(row["_loaded_at"] for row in full) # type: ignore[type-var] + + assert first == [row for row in full if row["_loaded_at"] == first_load] + assert len(first) < len(full) + later = [row for row in full if row not in first] + assert {row["_op"] for row in later} == {"I", "U", "D"} + shipments = {row["shipment_id"]: row["_seq"] for row in first} + assert any( + row["shipment_id"] in shipments and row["_seq"] < shipments[row["shipment_id"]] # type: ignore[operator] + for row in later + ), "the second load must carry a late older version" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index b668045bc4..7ea0c4a127 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1251,6 +1251,17 @@ class BenchmarkProjectCompileTestCase: expected_warnings: int +@dataclass(frozen=True) +class BenchmarkCoreCopyCompileTestCase: + """A script that copies a benchmark project without an area, and the folder it leaves out.""" + + description: str + copy_script: Path + expected_absent_folder: str + expected_errors: int + expected_warnings: int + + @dataclass(frozen=True) class GeneratedBenchmarkCompileTestCase: """A generated benchmark project, its recorded checksum and its cold compile budget.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py index eccd67e5fb..da59c60c45 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py @@ -5,12 +5,14 @@ import json import shutil import subprocess +import sys from pathlib import Path from typing import Any import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + BenchmarkCoreCopyCompileTestCase, BenchmarkProjectCompileTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT, run_sqb @@ -59,3 +61,42 @@ def test_given_benchmark_project_when_compiling_then_reports_no_errors_or_warnin assert summary["warnings"] == test_case.expected_warnings, diagnostics assert summary["models"] > 0 assert summary["tests"] > 0 + + +@pytest.mark.parametrize( + "test_case", + [ + BenchmarkCoreCopyCompileTestCase( + description="duckshop_core", + copy_script=REPO_ROOT / "benchmarks" / "duckshop" / "scripts" / "core_project.py", + expected_absent_folder="/edge_cases/", + expected_errors=0, + expected_warnings=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_benchmark_core_copy_when_compiling_then_leaves_out_the_excluded_area( + test_case: BenchmarkCoreCopyCompileTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / test_case.description + copied: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(test_case.copy_script), str(project_dir)], + capture_output=True, + text=True, + check=False, + ) + assert copied.returncode == 0, copied.stdout + copied.stderr + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stdout + result.stderr + report: dict[str, Any] = json.loads(result.stdout) + assert report["summary"]["errors"] == test_case.expected_errors, report["diagnostics"] + assert report["summary"]["warnings"] == test_case.expected_warnings, report["diagnostics"] + model_paths: str = "\n".join(model["relative_path"] for model in report["resources"]["models"]) + assert model_paths + assert test_case.expected_absent_folder not in model_paths From 07164db35b56eafba487e60a7b59e2f41f36fe45 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:54:15 +0000 Subject: [PATCH 6/6] test(benchmarks): pin DuckShop edge array order and guard the core copy by checksum Sort the results of ARRAY_EXCEPT, ARRAY_DISTINCT and OBJECT_KEYS, whose element order is unspecified, and recapture their expectations. The core-copy compile guard now compares the copy's compiled files with a recorded checksum, which core_project.py --checksum prints. The core copy also leaves out the edge Rule and its tests, renamed with an edge_ prefix, and the marked edge blocks of sqlbuild_project.toml. The Rule reads only the header's top-level tags, past strings holding ');'. The README and RUNBOOK say which expectations are captured and which are written by hand, that a schema built with the old identifier-case setting needs a clean rebuild, and that DEFAULT_DDL_COLLATION must stay at its default on the edge schema. --- benchmarks/duckshop/README.md | 9 ++- benchmarks/duckshop/RUNBOOK.md | 17 ++++- .../edge_cases/int_edge_payload_arrays.sql | 13 +++- .../edge_cases/int_edge_payload_objects.sql | 3 +- .../duckshop/project/rules/area_tags.py | 36 ---------- .../duckshop/project/rules/edge_area_tags.py | 68 +++++++++++++++++++ .../duckshop/project/sqlbuild_project.toml | 30 ++++---- ...le_area_tags.py => test_edge_area_tags.py} | 21 +++++- .../test_int_edge_payload_arrays.sql | 2 +- benchmarks/duckshop/scripts/core_project.py | 64 +++++++++++++++-- .../cli/commands/main/compile/_test_types.py | 7 +- .../test_duckshop_benchmark_compile.py | 11 +++ 12 files changed, 217 insertions(+), 64 deletions(-) delete mode 100644 benchmarks/duckshop/project/rules/area_tags.py create mode 100644 benchmarks/duckshop/project/rules/edge_area_tags.py rename benchmarks/duckshop/project/tests/{test_rule_area_tags.py => test_edge_area_tags.py} (73%) diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 745778dafd..f3166cfac9 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -129,15 +129,18 @@ groups: | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | -Every expected value in the edge-case area was captured on Snowflake under the `edge` target's -pinned session, rather than derived by hand. `fct_edge_delivery_events` is disabled: it is the +Unit-test expectations in the edge-case area were captured on Snowflake under the `edge` target's +pinned session, rather than derived by hand; scenario expectations, and the assertions in tests +about ties, are written by hand from those captured values. `fct_edge_delivery_events` is disabled: it is the acceptance case for cursor `delete_insert` models that emit rows outside the window they replace. See [RUNBOOK.md](RUNBOOK.md), step 7. Benchmarks and core measurements leave the area out: - `python benchmarks/duckshop/scripts/core_project.py ` copies the project without - it; the copy compiles to the same files as the project did before the area existed; + it. The core-copy compile guard compares the copy's compiled files with a recorded checksum + (`core_project.py --checksum`), so a change to the core project's compiled SQL fails the guard + until the checksum is updated on purpose and noted for the benchmark; - warehouse runs exclude it with `--exclude tag:edge_case "source:edge__*"`, and scenario runs with `--exclude edge_cases`; - the construct census counts the core project unless given `--area edge` or `--area all`. diff --git a/benchmarks/duckshop/RUNBOOK.md b/benchmarks/duckshop/RUNBOOK.md index 4e5605b5a7..6cd3759b96 100644 --- a/benchmarks/duckshop/RUNBOOK.md +++ b/benchmarks/duckshop/RUNBOOK.md @@ -54,6 +54,11 @@ no DuckShop names; if it does, stop and pick another schema. Note which state ta The list includes the edge-case area's names; run the check with `--target edge` too before its first build. +The project no longer sets `QUOTED_IDENTIFIERS_IGNORE_CASE = true` on its connections. A schema +built while it did may hold objects and SQLBuild state that no longer resolve under Snowflake's +default case rules, so rebuild into a clean schema: drop everything as in step 8, or pick a new +schema, before the first build. + ## 3. Load the synthetic data ```bash @@ -162,6 +167,11 @@ values were captured under: UTC, and Snowflake's defaults for every parameter th result or how text is parsed, including `QUOTED_IDENTIFIERS_IGNORE_CASE = false`. Run it on that target only; under another session some expected values differ by design. +`DEFAULT_DDL_COLLATION` also changes results (comparisons, ordering and `LIKE` on text columns), +but it is an account, database or schema parameter, not a session one, so the connection cannot +pin it. The edge target's schema, and its database and account, must leave it at the default (no +collation). Check with `SHOW PARAMETERS LIKE 'DEFAULT_DDL_COLLATION' IN SCHEMA .`. + ### Load, build and test ```bash @@ -202,8 +212,8 @@ chose. ### Capture expected values -Edge-case expectations are Snowflake's own results on the fixtures, not values derived by hand. -To capture them for a new or changed edge test, write its `__expected__` CTE with the +Unit-test expectations in the edge-case area are Snowflake's own results on the fixtures, not +values derived by hand. To capture them for a new or changed edge test, write its `__expected__` CTE with the columns to compare (any values, such as `NULL AS column_name`), then run: ```bash @@ -216,7 +226,8 @@ replaces each expected CTE with the actual rows as typed literals. Without a tes recaptures every edge test, which must leave the files unchanged. Check every captured value against the case the test describes before committing: a capture records what Snowflake returned, which is the point, but a fixture mistake is captured as faithfully as a real edge. Scenario -expectations are written by hand from the captured unit-test values. +expectations, and the assertions in tests about ties (where Snowflake may return any of the tied +rows), are written by hand from the captured values. Capture scenario snapshots for the edge scenarios with `sqb scenario capture --target edge --select edge_cases`. diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql index 9382ce1707..440b1acce2 100644 --- a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_arrays.sql @@ -37,12 +37,21 @@ final AS ( p.payload:gift_code::VARCHAR )::ARRAY AS codes_compact, ARRAY_COMPACT(p.payload:tags::ARRAY)::ARRAY AS tags_compact, - ARRAY_EXCEPT(p.payload:tags::ARRAY, p.payload:removed_tags::ARRAY)::ARRAY AS tags_kept, + -- ARRAY_EXCEPT and ARRAY_DISTINCT leave element order unspecified; sort ascending, NULLs last. + ARRAY_SORT( + ARRAY_EXCEPT(p.payload:tags::ARRAY, p.payload:removed_tags::ARRAY), + TRUE, + FALSE + )::ARRAY AS tags_kept, ARRAY_CAT( p.payload:tags::ARRAY, p.payload:extra_tags::ARRAY )::ARRAY AS tags_all, - ARRAY_DISTINCT(p.payload:tags::ARRAY)::ARRAY AS tags_distinct, + ARRAY_SORT( + ARRAY_DISTINCT(p.payload:tags::ARRAY), + TRUE, + FALSE + )::ARRAY AS tags_distinct, ARRAY_POSITION(p.payload:category_id, p.payload:eligible_ids::ARRAY)::INT AS category_position, ARRAY_CONTAINS(p.payload:category_id, p.payload:eligible_ids::ARRAY)::BOOLEAN AS is_eligible, ARRAYS_OVERLAP( diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql index c9735db890..3c0c1f17fe 100644 --- a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_payload_objects.sql @@ -75,7 +75,8 @@ final AS ( n.payload:pick_keys::ARRAY )::OBJECT AS picked_by_array, OBJECT_PICK(n.payload::OBJECT, n.payload:pick_keys)::OBJECT AS picked_by_variant, - OBJECT_KEYS(n.payload::OBJECT)::ARRAY AS payload_keys + -- OBJECT_KEYS leaves key order unspecified; sort ascending. + ARRAY_SORT(OBJECT_KEYS(n.payload::OBJECT), TRUE, FALSE)::ARRAY AS payload_keys FROM normalised AS n ) diff --git a/benchmarks/duckshop/project/rules/area_tags.py b/benchmarks/duckshop/project/rules/area_tags.py deleted file mode 100644 index bdb10fe45c..0000000000 --- a/benchmarks/duckshop/project/rules/area_tags.py +++ /dev/null @@ -1,36 +0,0 @@ -"""DuckShop house rule that keeps the edge-case area selectable on its own.""" - -import re - -from sqlbuild.rules import Finding, Model, RuleContext, rule - -EDGE_FOLDER = "edge_cases" -EDGE_TAG = "edge_case" -_HEADER_TAGS = re.compile(r"^[ \t]*tags\s*\[([^\]]*)\]", re.MULTILINE) - - -def _header_tags(sql: str) -> tuple[set[str], int]: - header_end = sql.find(");") - match = _HEADER_TAGS.search(sql, 0, header_end if header_end >= 0 else len(sql)) - if match is None: - return set(), 1 - tags = {tag.strip().strip("\"'") for tag in match.group(1).split(",") if tag.strip()} - return tags, sql[: match.start()].count("\n") + 1 - - -@rule( - code="XSQBRDUCK002", - message="Models in an edge_cases folder, and only those, must declare the edge_case tag", - remediation=( - "Add `tags [edge_case]` to the header of every model under models//edge_cases/, " - "and move a model that declares the tag into that folder, so benchmark and core runs can " - "exclude the edge-case area with --exclude tag:edge_case." - ), -) -def edge_area_is_tagged(*, model: Model, ctx: RuleContext) -> list[Finding]: - parts = ctx.project.tree.relative_parts(path=model.path, under="models") - in_area = EDGE_FOLDER in parts - tags, line = _header_tags(ctx.sql.for_model(model).authored.source) - if in_area == (EDGE_TAG in tags): - return [] - return [ctx.finding(subject=model, line=line)] diff --git a/benchmarks/duckshop/project/rules/edge_area_tags.py b/benchmarks/duckshop/project/rules/edge_area_tags.py new file mode 100644 index 0000000000..7ba35badac --- /dev/null +++ b/benchmarks/duckshop/project/rules/edge_area_tags.py @@ -0,0 +1,68 @@ +"""DuckShop house rule that keeps the edge-case area selectable on its own.""" + +import re + +from sqlbuild.rules import Finding, Model, RuleContext, rule + +EDGE_FOLDER = "edge_cases" +EDGE_TAG = "edge_case" +_TAGS_KEY = re.compile(r"tags\s*\[([^\]]*)\]") +_QUOTES = ('"""', '"', "'") + + +def _string_end(sql: str, start: int) -> int: + """The index after the string literal opening at `start`.""" + + quote = next(q for q in _QUOTES if sql.startswith(q, start)) + index = start + len(quote) + while index < len(sql): + if len(quote) == 1 and sql[index] == "\\": + index += 2 + elif sql.startswith(quote, index): + return index + len(quote) + else: + index += 1 + return len(sql) + + +def _header_tags(sql: str) -> tuple[set[str], int]: + """The tags the MODEL header declares at its top level, and their line.""" + + depth = 0 + index = sql.find("(") + while 0 <= index < len(sql): + char = sql[index] + if char in "\"'": + index = _string_end(sql, index) + continue + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + break + elif depth == 1 and sql.startswith("tags", index) and not sql[index - 1].isalnum(): + match = _TAGS_KEY.match(sql, index) + if match is not None: + tags = {t.strip().strip("\"'") for t in match.group(1).split(",") if t.strip()} + return tags, sql[:index].count("\n") + 1 + index += 1 + return set(), 1 + + +@rule( + code="XSQBRDUCK002", + message="Models in an edge_cases folder, and only those, must declare the edge_case tag", + remediation=( + "Add `tags [edge_case]` to the header of every model under models//edge_cases/, " + "and move a model that declares the tag into that folder, so benchmark and core runs can " + "exclude the edge-case area with --exclude tag:edge_case." + ), +) +def edge_area_is_tagged(*, model: Model, ctx: RuleContext) -> list[Finding]: + parts = ctx.project.tree.relative_parts(path=model.path, under="models") + in_area = EDGE_FOLDER in parts + tags, line = _header_tags(ctx.sql.for_model(model).authored.source) + if in_area == (EDGE_TAG in tags): + return [] + return [ctx.finding(subject=model, line=line)] diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 9dd0e90060..d93428c719 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -10,9 +10,20 @@ user = "example_user" role = "EXAMPLE_ROLE" warehouse = "EXAMPLE_WH" +[targets.dev] +connection = "warehouse" +database = "DUCKSHOP" +schema = "DEV" + +[targets.prod] +connection = "warehouse" +database = "DUCKSHOP" +schema = "ANALYTICS" + +# >>> edge-case area (scripts/core_project.py leaves this block out of the core copy) # The edge-case area's expected values are captured on Snowflake under this pinned session: UTC, # with every parameter that changes a result or how text is parsed set to Snowflake's default. -# Run the edge area on this connection only (see RUNBOOK.md, "Edge cases"). +# Run the edge area on this connection only (see RUNBOOK.md, step 7). [connections.edge] account = "example-account" user = "example_user" @@ -43,20 +54,11 @@ TIMESTAMP_DAY_IS_ALWAYS_24H = false JSON_INDENT = 2 STRICT_JSON_OUTPUT = false -[targets.dev] -connection = "warehouse" -database = "DUCKSHOP" -schema = "DEV" - [targets.edge] connection = "edge" database = "DUCKSHOP" schema = "EDGE" - -[targets.prod] -connection = "warehouse" -database = "DUCKSHOP" -schema = "ANALYTICS" +# <<< edge-case area [settings] default_audit_severity = "error" @@ -90,9 +92,11 @@ valuation_date = "2026-03-31" lookup_customer_id = "C000001" # Volume multiplier for the synthetic data loaders in python/loaders; 1 is the small default. data_scale = "1" +# >>> edge-case area (scripts/core_project.py leaves this block out of the core copy) # Last load of the edge-case delivery change log to include; "1" replays the first load only (see -# RUNBOOK.md, "Edge cases"). +# RUNBOOK.md, step 7). edge_delivery_batch = "2" +# <<< edge-case area [scenario.local_type_overrides.snowflake] "VARIANT" = "JSON" @@ -273,6 +277,7 @@ The KPI grid is every reporting day for every region and metric pair that has ev days without activity report zero instead of disappearing. Both sides are small and bounded. """ +# >>> edge-case area (scripts/core_project.py leaves this block out of the core copy) [[rules.rule_ignores]] rules = ["SQBRSQL027"] paths = ["models/marts/edge_cases/rpt_edge_stock_reconciliation.sql"] @@ -298,3 +303,4 @@ Unit-test and scenario fixtures for a source whose declared columns are quoted c rejected (B302 and S510), so this model is covered by building the edge-case area on Snowflake, where it reads the loaded quoted columns, instead of by a unit test. """ +# <<< edge-case area diff --git a/benchmarks/duckshop/project/tests/test_rule_area_tags.py b/benchmarks/duckshop/project/tests/test_edge_area_tags.py similarity index 73% rename from benchmarks/duckshop/project/tests/test_rule_area_tags.py rename to benchmarks/duckshop/project/tests/test_edge_area_tags.py index cebc6af87e..77eed8f3cf 100644 --- a/benchmarks/duckshop/project/tests/test_rule_area_tags.py +++ b/benchmarks/duckshop/project/tests/test_edge_area_tags.py @@ -1,4 +1,4 @@ -from rules.area_tags import edge_area_is_tagged +from rules.edge_area_tags import edge_area_is_tagged from sqlbuild.rules import RuleCase, evaluate_rule @@ -64,3 +64,22 @@ def test_given_untagged_model_outside_edge_folder_when_evaluating_then_passes() ) assert result.finding_count == 0 + + +def test_given_tags_text_in_description_when_evaluating_then_reads_only_header_tags() -> None: + result = evaluate_rule( + rule=edge_area_is_tagged, + test_case=RuleCase( + description="a description holding ');' and tags text", + source=( + 'MODEL (\n description """Orders (see note);\n tags [edge_case]""",\n' + " columns (\n order_id (type VARCHAR),\n ),\n" + " tags [orders],\n);\nSELECT 1 AS order_id\n" + ), + path="models/staging/edge_cases/stg_edge_orders.sql", + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + assert result.findings[0].line == 7 diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql index daaf0aa323..cd55a2274c 100644 --- a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_payload_arrays.sql @@ -26,7 +26,7 @@ __expected__int_edge_payload_arrays AS ( PARSE_JSON('["SPRING",undefined]')::ARRAY AS codes, PARSE_JSON('["SPRING"]')::ARRAY AS codes_compact, PARSE_JSON('["bath","gift","bath","bath"]')::ARRAY AS tags_compact, - PARSE_JSON('["gift","bath",null,"bath"]')::ARRAY AS tags_kept, + PARSE_JSON('["bath","bath","gift",null]')::ARRAY AS tags_kept, PARSE_JSON('["bath","gift","bath",null,"bath","gift",null]')::ARRAY AS tags_all, PARSE_JSON('["bath","gift",null]')::ARRAY AS tags_distinct, TRUE AS shares_extra_tags diff --git a/benchmarks/duckshop/scripts/core_project.py b/benchmarks/duckshop/scripts/core_project.py index 68bd4d8438..0bf2b000d2 100644 --- a/benchmarks/duckshop/scripts/core_project.py +++ b/benchmarks/duckshop/scripts/core_project.py @@ -1,23 +1,55 @@ """Copy the DuckShop project without its edge-case area, for benchmarks and core measurements. python benchmarks/duckshop/scripts/core_project.py + python benchmarks/duckshop/scripts/core_project.py --checksum The edge-case area holds hand-picked edge values, not benchmark workload, so compile benchmarks -measure this copy. It leaves out every file and folder whose name starts with `edge_` or `_edge_`: -the `edge_cases` model, test and scenario folders, their source file and loaders, and their -scenario snapshots. Local-only files such as `target/` and `sqlbuild_local.toml` are left out too. +measure this copy. It leaves out every file and folder whose name starts with `edge_`, `_edge_` +or `test_edge_`: the `edge_cases` model, test and scenario folders, their source file, loaders, +Rule and Rule tests, and their scenario snapshots. From `sqlbuild_project.toml` it leaves out the +blocks between `# >>> edge-case area` and `# <<< edge-case area`: the edge connection and target, +the edge variable and the edge Rule ignores. Local-only files such as `target/` and +`sqlbuild_local.toml` are left out too. + +`--checksum` prints the SHA-256 of a compiled project's `target/compiled/` files, the value the +core-copy compile guard compares with. A change to a core model changes it on purpose; record the +new value in the guard and note the change for the benchmark. """ from __future__ import annotations import argparse +import hashlib import pathlib import shutil import sys PROJECT = pathlib.Path(__file__).resolve().parent.parent / "project" -EDGE_AREA_PATTERNS = ("edge_*", "_edge_*") +PROJECT_FILE = "sqlbuild_project.toml" +EDGE_AREA_PATTERNS = ("edge_*", "_edge_*", "test_edge_*") LOCAL_ONLY_PATTERNS = ("target", "logs", ".sqlbuild", "sqlbuild_local.toml", "__pycache__") +EDGE_BLOCK_START = "# >>> edge-case area" +EDGE_BLOCK_END = "# <<< edge-case area" + + +def without_edge_blocks(text: str) -> str: + """The project file without its marked edge-case blocks, markers included.""" + + kept: list[str] = [] + inside = False + for line in text.splitlines(keepends=True): + if line.startswith(EDGE_BLOCK_START): + inside = True + elif line.startswith(EDGE_BLOCK_END): + inside = False + elif not inside: + kept.append(line) + if inside: + raise ValueError(f"{PROJECT_FILE}: '{EDGE_BLOCK_START}' has no '{EDGE_BLOCK_END}'") + text = "".join(kept) + while "\n\n\n" in text: + text = text.replace("\n\n\n", "\n\n") + return text.rstrip("\n") + "\n" def copy_core_project(destination: pathlib.Path) -> None: @@ -26,12 +58,36 @@ def copy_core_project(destination: pathlib.Path) -> None: destination, ignore=shutil.ignore_patterns(*EDGE_AREA_PATTERNS, *LOCAL_ONLY_PATTERNS), ) + project_file = destination / PROJECT_FILE + project_file.write_text(without_edge_blocks(project_file.read_text())) + + +def compiled_checksum(project_dir: pathlib.Path) -> str: + """SHA-256 over every compiled file's relative path and bytes, in path order.""" + + compiled = project_dir / "target" / "compiled" + digest = hashlib.sha256() + for path in sorted(p for p in compiled.rglob("*") if p.is_file()): + digest.update(path.relative_to(compiled).as_posix().encode() + b"\0") + digest.update(path.read_bytes() + b"\0") + return digest.hexdigest() def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("destination", type=pathlib.Path, help="new folder for the copy") + parser.add_argument( + "--checksum", + action="store_true", + help="print the checksum of DESTINATION's compiled files instead of copying", + ) args = parser.parse_args() + if args.checksum: + if not (args.destination / "target" / "compiled").is_dir(): + print(f"{args.destination} has no target/compiled; compile it first", file=sys.stderr) + return 1 + print(compiled_checksum(args.destination)) + return 0 if args.destination.exists(): print(f"{args.destination} already exists", file=sys.stderr) return 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7ea0c4a127..c774b30615 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1253,13 +1253,18 @@ class BenchmarkProjectCompileTestCase: @dataclass(frozen=True) class BenchmarkCoreCopyCompileTestCase: - """A script that copies a benchmark project without an area, and the folder it leaves out.""" + """A script that copies a benchmark project without an area, and the copy's compiled output. + + `expected_compiled_checksum` is the script's `--checksum` of the copy's compiled files; a core + model change updates it deliberately. + """ description: str copy_script: Path expected_absent_folder: str expected_errors: int expected_warnings: int + expected_compiled_checksum: str @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py index da59c60c45..f93c80252f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py @@ -72,6 +72,9 @@ def test_given_benchmark_project_when_compiling_then_reports_no_errors_or_warnin expected_absent_folder="/edge_cases/", expected_errors=0, expected_warnings=0, + expected_compiled_checksum=( + "4568ab62e8033a5698793db6993f19467ff68bd919c3ae2e2f72055b8ac2cde0" + ), ) ], ids=lambda case: case.description, @@ -100,3 +103,11 @@ def test_given_benchmark_core_copy_when_compiling_then_leaves_out_the_excluded_a model_paths: str = "\n".join(model["relative_path"] for model in report["resources"]["models"]) assert model_paths assert test_case.expected_absent_folder not in model_paths + checksum: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(test_case.copy_script), "--checksum", str(project_dir)], + capture_output=True, + text=True, + check=False, + ) + assert checksum.returncode == 0, checksum.stdout + checksum.stderr + assert checksum.stdout.strip() == test_case.expected_compiled_checksum