From 1b78877a5a8eea0fe6051816aca902c2da366b44 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:39:21 +0000 Subject: [PATCH 1/5] feat(duckshop): generate a DuckDB copy of core DuckShop for demonstrations The script compiles the core project, translates each compiled model from Snowflake to DuckDB with Polyglot plus DuckShop-specific rewrites, and writes Snowflake functions DuckDB lacks as project SQL macros. The copy compiles with no errors and builds every model on DuckDB; an E2E test runs the whole path. --- benchmarks/duckshop/README.md | 6 + benchmarks/duckshop/scripts/duckdb_project.py | 264 ++++++++ .../duckshop/scripts/duckdb_rewrites.py | 602 ++++++++++++++++++ .../cli/commands/main/compile/_test_types.py | 10 + .../test_duckshop_benchmark_compile.py | 45 ++ 5 files changed, 927 insertions(+) create mode 100644 benchmarks/duckshop/scripts/duckdb_project.py create mode 100644 benchmarks/duckshop/scripts/duckdb_rewrites.py diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index fd9f67e96a..505f536c45 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -155,6 +155,12 @@ Benchmarks and core measurements leave the area out: with `--exclude edge_cases`; - the construct census counts the core project unless given `--area edge` or `--area all`. +`uv run --script benchmarks/duckshop/scripts/duckdb_project.py ` writes a DuckDB copy +of the core project for demonstrations: it compiles the core copy, translates each compiled model +with Polyglot and the rewrites in `duckdb_rewrites.py`, and leaves out tests, scenarios, custom +Rules and Snowflake-only hooks. `sqb build` in the copy loads the synthetic data and builds every +model on DuckDB, which an E2E test checks. It is not a benchmark workload. + Changes to the core project's compiled SQL, after which benchmark results are not comparable with earlier ones: diff --git a/benchmarks/duckshop/scripts/duckdb_project.py b/benchmarks/duckshop/scripts/duckdb_project.py new file mode 100644 index 0000000000..8b02cab5de --- /dev/null +++ b/benchmarks/duckshop/scripts/duckdb_project.py @@ -0,0 +1,264 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = ["polyglot-sql-chio==0.13.10", "pyyaml"] +# /// +"""Write a DuckDB copy of core DuckShop, translated from its compiled Snowflake models. + + uv run --script benchmarks/duckshop/scripts/duckdb_project.py + +Needs `sqb` on PATH. The script copies the core project (`core_project.py`), compiles it, and +writes a DuckDB project to DESTINATION: + +- Each model keeps its header, with Snowflake types mapped to DuckDB's, and gets its compiled SQL + translated by Polyglot and finished by `duckdb_rewrites.py`. Macros, constants and private + declarations are already inlined there, so the copy drops them; it keeps the enums a column is + typed with. +- Snowflake functions DuckDB lacks become project SQL macros, written where their callers are. +- Sources, seeds, schemas, audits and the synthetic data loaders are adapted for DuckDB. +- Tests, scenarios, custom Rules and Snowflake-only hooks are left out. + +The copy is for demonstrations: build it with `sqb load` and `sqb build` on DuckDB. It is not a +benchmark workload, and the Snowflake project stays the source of truth. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any + +import polyglot_sql +import yaml +from core_project import copy_core_project +from duckdb_rewrites import balanced, convert, shim_calls, write_shims + +TYPE_MAP = [ + (r"\bNUMBER\(38, ?0\)|\b(INT|INTEGER|SMALLINT|TINYINT)\b", "BIGINT"), + (r"\b(VARIANT|OBJECT|ARRAY)\b", "JSON"), + (r"\bTIMESTAMP_NTZ\b", "TIMESTAMP"), + (r"\bTIMESTAMP_(TZ|LTZ)\b", "TIMESTAMPTZ"), + (r"\bNUMBER\b", "DECIMAL"), +] +CONNECTION = """[connections.warehouse] +database = "duckshop.duckdb" + +[targets.dev] +connection = "warehouse" +schema = "dev" + +[targets.prod] +connection = "warehouse" +schema = "analytics" +""" +WRITER_QUOTE = ( + ' return "\'" + text.replace("\\\\", "\\\\\\\\").replace("\'", "\'\'") + "\'"', + ' return "\'" + text.replace("\'", "\'\'") + "\'"', +) +WRITER_INSERT = '''def insert_sql( + destination: str, columns: tuple[tuple[str, str], ...], rows: list[dict[str, object]] +) -> str: + """One INSERT for a batch: typed casts happen in the select list.""" + + names = ", ".join(name for name, _ in columns) + aliases = ", ".join(f"column{index}" for index in range(1, len(columns) + 1)) + select = ", ".join( + f"CAST(column{index} AS {sql_type})" for index, (_, sql_type) in enumerate(columns, 1) + ) + values = ",\\n".join( + "(" + ", ".join(literal(row[name]) for name, _ in columns) + ")" for row in rows + ) + rows_sql = f"FROM (VALUES\\n{values}) AS v({aliases})" + return f"INSERT INTO {destination} ({names})\\nSELECT {select}\\n{rows_sql}" + + +def statements''' + + +def types(text: str) -> str: + """Snowflake types as DuckDB's. Snowflake's integers are all NUMBER(38, 0): BIGINT here.""" + + for pattern, new in TYPE_MAP: + text = re.sub(pattern, new, text) + return text + + +def translate(sql: str, model: str | None = None) -> str: + # GET with a computed key and REGEXP_SUBSTR, which returns NULL rather than '' when nothing + # matches, go to their shims untranslated: Polyglot's DuckDB forms behave differently. + sql = re.sub(r"(? str: + m = re.match(rf"\s*{keyword} \(.*?^\);\n", text, re.S | re.M) + if m is None: + raise ValueError(f"no {keyword} header") + return m.group(0) + + +def audit_sql(text: str) -> str: + """Audit SQL in headers and audit files, with Snowflake's IFF as DuckDB's IF.""" + + return re.sub(r"(? str: + """A model header without private constants and enums, which its compiled SQL inlines, and + without Snowflake-only hooks.""" + + head = header(text, "MODEL") + head = re.sub(r'(?m)^ +inline_sql\("[^"]*CHANGE_TRACKING[^"]*"\),\n', "", head) + head = re.sub(r"(?m)^ post_hooks \[\n \],\n", "", head) + for block in ("constants", "enums"): + m = re.search(rf"^ {block} \(", head, re.M) + if m: + end = balanced(head, m.end()) + head = head[: m.start()] + head[end:].lstrip(",").lstrip("\n") + return audit_sql(types(head)) + + +def compile_snowflake(project: pathlib.Path) -> dict[str, Any]: + result = subprocess.run( + ["sqb", "compile", "--json"], cwd=project, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + sys.stderr.write(result.stderr) + raise SystemExit("compiling the Snowflake project failed") + return json.loads(result.stdout) + + +def write_models(snow: pathlib.Path, out: pathlib.Path, report: dict[str, Any]) -> None: + for model in report["resources"]["models"]: + path = out / model["relative_path"] + body = translate(model["query_sql"], model["name"]) + path.write_text(model_header(path.read_text()) + "\n" + body + "\n") + for path in (out / "functions" / "sql").glob("*.sql"): + text = path.read_text() + head = header(text, "FUNCTION") + compiled = snow / "target" / "compiled" / "functions" / path.name + body = compiled.read_text() if compiled.exists() else text[len(head) :] + path.write_text(types(head) + "\n" + translate(body) + "\n") + + +def write_sources(out: pathlib.Path) -> None: + for path in [*(out / "sources").glob("*.yml"), out / "seeds" / "seeds.yml"]: + doc = yaml.safe_load(path.read_text()) + for entry in doc.get("sources", doc.get("seeds", [])): + for column in entry.get("columns", []): + column["type"] = types(column["type"]) + settings = entry.get("csv_settings") or {} + if isinstance(settings.get("na_values"), dict): + # DuckDB seeds take one null list for every column. + values = {str(v) for vs in settings["na_values"].values() for v in vs} + settings["na_values"] = sorted(values) + if entry.get("expression"): + entry["expression"] = translate(entry["expression"]) + "\n" + path.write_text(yaml.safe_dump(doc, sort_keys=False, width=100)) + for path in out.rglob("*.py"): + if "functions" not in path.parts: + path.write_text(types(path.read_text())) + writer = out / "python" / "loaders" / "_writer.py" + text = writer.read_text() + if WRITER_QUOTE[0] not in text: + raise SystemExit("python/loaders/_writer.py changed: update WRITER_QUOTE") + text = text.replace(*WRITER_QUOTE) + text, count = re.subn( + r"def insert_sql\(.*?\n\n\ndef statements", lambda _: WRITER_INSERT, text, flags=re.S + ) + if count != 1: + raise SystemExit("python/loaders/_writer.py changed: update WRITER_INSERT") + writer.write_text(text) + + +def write_declarations(out: pathlib.Path) -> None: + """Keep enums that type a column, owned by their columns' folder; drop other declarations, + which compiled model SQL inlines. Integer enums become BIGINT columns: DuckDB types their + members INTEGER.""" + + for path in [p for p in out.rglob("*.sql") if "audits" in str(p.relative_to(out))]: + path.write_text(audit_sql(path.read_text())) + for path in out.glob("models/**/_sqlbuild/_schemas/*.sql"): + path.write_text(types(path.read_text())) + models = [p for p in out.glob("models/**/*.sql") if "_sqlbuild" not in p.parts] + enums = {path.stem: path for path in out.glob("**/enums/*.sql")} + for name in [n for n, p in enums.items() if re.search(r"members \(\w+ \d", p.read_text())]: + del enums[name] + for path in models: + path.write_text(re.sub(rf"\(type {name}\b", "(type BIGINT", path.read_text())) + owners: dict[str, set[pathlib.Path]] = {} + for path in models: + for name in re.findall(r"\(type (\w+)\b", header(path.read_text(), "MODEL")): + if name in enums: + owners.setdefault(name, set()).add(path.parent) + keep: dict[pathlib.Path, str] = {} + for name, folders in owners.items(): + folder = folders.pop() / "_sqlbuild" / "_enums" if len(folders) == 1 else out / "enums" + keep[folder / f"{name}.sql"] = types(enums[name].read_text()) + for gone in ("tests", "macros", "rules", "constants", "enums"): + shutil.rmtree(out / gone, ignore_errors=True) + for folder in ("_macros", "macros", "constants", "_constants", "enums", "_enums"): + for path in list(out.glob(f"models/**/_sqlbuild/{folder}")): + shutil.rmtree(path) + for path, text in keep.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + write_shims(out) + + +def write_project_file(out: pathlib.Path) -> None: + path = out / "sqlbuild_project.toml" + text = path.read_text().replace('adapter = "snowflake"', 'adapter = "duckdb"') + start = text.index("[connections.warehouse]") + end = text.index("\n\n", text.index("[targets.prod]")) + text = text[:start] + CONNECTION + text[end:] + text = re.sub(r"select = \[[^\]]*\]", 'select = ["SQBRGRAPH", "SQBRCONTRACT"]', text, count=1) + text = re.sub(r"(?ms)^\[scenario\.local_type_overrides\.snowflake\]\n.*?(?=^\[)", "", text) + # Rule exceptions for files the copy leaves out, such as macros, would fail compilation. + head, *blocks = re.split(r"(?m)^(?=\[\[rules\.rule_exceptions\]\])", text) + kept = [] + for block in blocks: + m = re.search(r'^path = "([^"]+)"', block, re.M) + if not m or (out / m.group(1)).exists(): + kept.append(block) + path.write_text(head + "".join(kept)) + (out / "sqlbuild_local.toml.example").unlink(missing_ok=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Write a DuckDB copy of core DuckShop.") + parser.add_argument("destination", type=pathlib.Path, help="new folder for the DuckDB copy") + args = parser.parse_args() + out: pathlib.Path = args.destination + if out.exists(): + print(f"{out} already exists", file=sys.stderr) + return 1 + with tempfile.TemporaryDirectory() as scratch: + snow = pathlib.Path(scratch) / "snowflake" + copy_core_project(snow) + report = compile_snowflake(snow) + shutil.copytree(snow, out, ignore=shutil.ignore_patterns("target", ".sqlbuild", "logs")) + write_models(snow, out, report) + write_sources(out) + write_declarations(out) + write_project_file(out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/duckshop/scripts/duckdb_rewrites.py b/benchmarks/duckshop/scripts/duckdb_rewrites.py new file mode 100644 index 0000000000..f9b936c9c7 --- /dev/null +++ b/benchmarks/duckshop/scripts/duckdb_rewrites.py @@ -0,0 +1,602 @@ +"""SQL rewrites that finish Polyglot's Snowflake-to-DuckDB translation of DuckShop's models. + +Used by `duckdb_project.py`. Polyglot translates most Snowflake SQL; these rules cover what it +leaves out or translates with different semantics. Semi-structured values are JSON throughout: +Snowflake's VARIANT, OBJECT and ARRAY all become DuckDB JSON. Snowflake functions DuckDB lacks +become project SQL macros named `sf_`, one per argument count. +""" + +from __future__ import annotations + +import os +import pathlib +import re +from collections.abc import Callable + +_UNQUOTED = "TRIM(CAST(x AS VARCHAR), '\"')" +_SNOWFLAKE_DATE_PARTS = ( + "replace(replace(replace(replace(replace(replace(f, 'YYYY', '%Y'), 'MM', '%m'), 'DD', '%d'), " + "'HH24', '%H'), 'MI', '%M'), 'SS', '%S')" +) + +# Each shim's overloads as (parameters, body). A shim with one overload keeps its own name; others +# get the argument count as a suffix, such as sf_to_number_3. +SHIMS: dict[str, list[tuple[str, str]]] = { + "sf_strip_null_value": [("x", "CASE WHEN CAST(x AS VARCHAR) = 'null' THEN NULL ELSE x END")], + "sf_try_to_number": [ + ("x", f"CAST(TRY_CAST({_UNQUOTED} AS DECIMAL(38, 0)) AS BIGINT)"), + ("x, p", f"CAST(TRY_CAST({_UNQUOTED} AS DECIMAL(38, 0)) AS BIGINT)"), + ("x, p, s", f"TRY_CAST({_UNQUOTED} AS DECIMAL(38, 9))"), + ], + "sf_get_path": [("j, p", "CAST(j AS JSON) -> ('$.' || p)")], + "sf_to_timestamp_ntz": [("x", f"TRY_CAST({_UNQUOTED} AS TIMESTAMP)")], + "sf_try_to_timestamp_tz": [ + ("x", f"TRY_CAST({_UNQUOTED} AS TIMESTAMPTZ)"), + ("x, f", f"TRY_CAST({_UNQUOTED} AS TIMESTAMPTZ)"), + ], + "sf_object_keys": [("j", "json_keys(CAST(j AS JSON))")], + "sf_is_object": [("j", "json_type(CAST(j AS JSON)) = 'OBJECT'")], + "sf_initcap": [ + ( + "x", + "array_to_string(list_transform(string_split(lower(x), ' '), " + "lambda w: upper(w[1]) || w[2:]), ' ')", + ) + ], + "sf_array_size": [("x", "CAST(json_array_length(to_json(x)) AS BIGINT)")], + "sf_regexp_substr": [ + ("s, p", "NULLIF(regexp_extract(s, p), '')"), + ("s, p, pos", "NULLIF(regexp_extract(s, p), '')"), + ("s, p, pos, occ", "NULLIF(regexp_extract(s, p), '')"), + ("s, p, pos, occ, flags", "NULLIF(regexp_extract(s, p), '')"), + ("s, p, pos, occ, flags, grp", "NULLIF(regexp_extract(s, p, grp), '')"), + ], + "sf_to_varchar": [ + ("x", _UNQUOTED), + ("x, f", f"strftime(x, {_SNOWFLAKE_DATE_PARTS})"), + ], + "sf_to_number": [ + ("x", f"CAST(CAST({_UNQUOTED} AS DECIMAL(38, 0)) AS BIGINT)"), + ("x, p", f"CAST(CAST({_UNQUOTED} AS DECIMAL(38, 0)) AS BIGINT)"), + ("x, p, s", f"CAST({_UNQUOTED} AS DECIMAL(38, 9))"), + ], + "sf_parse_url": [ + ( + "u, p", + "{'host': regexp_extract(u, '^[a-zA-Z]+://([^/:?#]+)', 1), " + "'path': regexp_extract(u, '^[a-zA-Z]+://[^/?#]*(/[^?#]*)?', 1), " + "'parameters': map_from_entries(list_transform(list_filter(" + "string_split(regexp_extract(u, '[?]([^#]*)', 1), '&'), lambda kv: kv <> ''), " + "lambda kv: {'key': split_part(kv, '=', 1), 'value': split_part(kv, '=', 2)})), " + "'query': regexp_extract(u, '\\?([^#]*)', 1), " + "'scheme': regexp_extract(u, '^([a-zA-Z]+)://', 1)}", + ) + ], + "sf_is_decimal": [("x", "json_type(to_json(x)) IN ('DOUBLE', 'UBIGINT', 'BIGINT')")], + "sf_is_integer": [("x", "json_type(to_json(x)) IN ('UBIGINT', 'BIGINT')")], + "sf_typeof": [ + ( + "x", + "CASE json_type(to_json(x)) WHEN 'VARCHAR' THEN 'VARCHAR' WHEN 'BIGINT' THEN 'INTEGER' " + "WHEN 'UBIGINT' THEN 'INTEGER' WHEN 'DOUBLE' THEN 'DECIMAL' " + "WHEN 'BOOLEAN' THEN 'BOOLEAN' WHEN 'OBJECT' THEN 'OBJECT' " + "WHEN 'ARRAY' THEN 'ARRAY' ELSE 'NULL_VALUE' END", + ) + ], + "sf_array_contains": [("a, v", "list_contains(CAST(to_json(a) AS JSON[]), to_json(v))")], + "sf_array_flatten": [("x", "flatten(CAST(to_json(x) AS JSON[][]))")], + "sf_try_to_timestamp_ntz": [ + ("x", f"TRY_CAST({_UNQUOTED} AS TIMESTAMP)"), + ("x, f", f"TRY_CAST({_UNQUOTED} AS TIMESTAMP)"), + ], + "sf_to_timestamp_tz": [ + ("x", f"CAST({_UNQUOTED} AS TIMESTAMPTZ)"), + ("x, f", f"CAST({_UNQUOTED} AS TIMESTAMPTZ)"), + ], + "sf_regexp_like": [("s, p", "regexp_full_match(s, p)"), ("s, p, f", "regexp_full_match(s, p)")], + "sf_array_distinct": [("x", "list_distinct(CAST(to_json(x) AS JSON[]))")], + "sf_sha2": [("x", "sha256(CAST(x AS VARCHAR))"), ("x, n", "sha256(CAST(x AS VARCHAR))")], + "sf_dateadd": [("u, n, d", "d + CAST(CAST(n AS VARCHAR) || ' ' || u AS INTERVAL)")], + "sf_get": [ + ( + "j, k", + "CASE WHEN json_type(CAST(j AS JSON)) = 'ARRAY' THEN CAST(j AS JSON) -> " + "TRY_CAST(k AS BIGINT) ELSE CAST(j AS JSON) -> CAST(k AS VARCHAR) END", + ) + ], + "sf_object_agg": [ + ( + "k, v", + "to_json(map_from_entries(ARRAY_AGG({'key': CAST(k AS VARCHAR), 'value': to_json(v)}) " + "FILTER (WHERE k IS NOT NULL AND v IS NOT NULL)))", + ) + ], + "sf_hash": [("x", "CAST(hash(x) >> 1 AS BIGINT)")], + "sf_list_count": [ + ( + "a", + "len(list_filter(CAST(to_json(a) AS JSON[]), lambda e: CAST(e AS VARCHAR) <> 'null'))", + ) + ], +} +# Declared result types, by shim or by shim and argument count. +RETURNS: dict[str, str] = { + "sf_strip_null_value": "JSON", + "sf_try_to_number": "BIGINT", + "sf_try_to_number_3": "DECIMAL(38, 9)", + "sf_to_number": "BIGINT", + "sf_to_number_3": "DECIMAL(38, 9)", + "sf_get_path": "JSON", + "sf_get": "JSON", + "sf_to_timestamp_ntz": "TIMESTAMP", + "sf_try_to_timestamp_ntz": "TIMESTAMP", + "sf_try_to_timestamp_tz": "TIMESTAMPTZ", + "sf_to_timestamp_tz": "TIMESTAMPTZ", + "sf_object_keys": "VARCHAR[]", + "sf_is_object": "BOOLEAN", + "sf_initcap": "VARCHAR", + "sf_array_size": "BIGINT", + "sf_regexp_substr": "VARCHAR", + "sf_to_varchar": "VARCHAR", + "sf_is_decimal": "BOOLEAN", + "sf_is_integer": "BOOLEAN", + "sf_typeof": "VARCHAR", + "sf_array_contains": "BOOLEAN", + "sf_array_flatten": "JSON[]", + "sf_regexp_like": "BOOLEAN", + "sf_array_distinct": "JSON[]", + "sf_sha2": "VARCHAR", + "sf_list_count": "BIGINT", + "sf_object_agg": "JSON", + "sf_hash": "BIGINT", +} +# Snowflake functions as Polyglot leaves them, and their shims or DuckDB equivalents. +RENAMES: dict[str, str] = { + "STRIP_NULL_VALUE": "sf_strip_null_value", + "TRY_TO_NUMBER": "sf_try_to_number", + "GET_PATH": "sf_get_path", + "TO_TIMESTAMP_NTZ": "sf_to_timestamp_ntz", + "TRY_TO_TIMESTAMP_TZ": "sf_try_to_timestamp_tz", + "OBJECT_KEYS": "sf_object_keys", + "IS_OBJECT": "sf_is_object", + "INITCAP": "sf_initcap", + "ARRAY_LENGTH": "sf_array_size", + "ARRAY_SIZE": "sf_array_size", + "ARRAY_CONSTRUCT": "json_array", + "OBJECT_CONSTRUCT": "json_object", + "REGEXP_SUBSTR": "sf_regexp_substr", + "TO_VARCHAR": "sf_to_varchar", + "TO_NUMBER": "sf_to_number", + "PARSE_URL": "sf_parse_url", + "OBJECT_AGG": "sf_object_agg", + "HASH": "sf_hash", + "JSON_GROUP_OBJECT": "sf_object_agg", + "IS_DECIMAL": "sf_is_decimal", + "IS_INTEGER": "sf_is_integer", + "TYPEOF": "sf_typeof", + "ARRAY_CONTAINS": "sf_array_contains", + "LIST_COUNT": "sf_list_count", + "ARRAY_DISTINCT": "sf_array_distinct", + "LIST_DISTINCT": "sf_array_distinct", + "SHA2": "sf_sha2", + "DATEADD": "sf_dateadd", + "FLATTEN": "sf_array_flatten", + "TRY_TO_TIMESTAMP_NTZ": "sf_try_to_timestamp_ntz", + "TO_TIMESTAMP_TZ": "sf_to_timestamp_tz", + "REGEXP_LIKE": "sf_regexp_like", +} +LIST_FUNCTIONS = { + "LIST_TRANSFORM": "list_transform", + "TRANSFORM": "list_transform", + "LIST_FILTER": "list_filter", + "FILTER": "list_filter", + "LIST_MIN": "list_min", + "LIST_MAX": "list_max", +} +FLATTEN_COLUMNS = re.compile( + r'\)\s+AS\s+(\w+)\(\s*SEQ,\s*KEY,\s*PATH,\s*"INDEX",\s*VALUE,\s*THIS\s*\)' +) +SNOWFLAKE_FORMATS = [ + ("YYYY", "%Y"), + ("HH24", "%H"), + ("MON", "%b"), + ("MM", "%m"), + ("DD", "%d"), + ("DY", "%a"), + ("MI", "%M"), + ("SS", "%S"), +] +# Columns that carry JSON values through a CTE, where a text cast must unquote them, by model. +JSON_COLUMNS = { + "int_product_specs_flat": {"pv.material", "pv.colour_family", "pv.dishwasher_safe"}, + "int_survey_answers_flat": {"t.answer_value"}, +} +JSON_FUNCTIONS = ("sf_strip_null_value", "sf_get", "sf_get_path", "to_json") + + +def balanced(text: str, start: int) -> int: + """Index just past the parenthesis that closes the one opened before `start`.""" + + depth, i, quote = 1, start, None + while depth: + c = text[i] + if quote: + if c == quote: + quote = None + elif c in "'\"": + quote = c + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + return i + + +def split_args(text: str) -> list[str]: + """Top-level comma-separated arguments, keeping a lambda's parameters together.""" + + args, depth, quote, start = [], 0, None, 0 + for i, c in enumerate(text): + if quote: + if c == quote: + quote = None + elif c in "'\"": + quote = c + elif c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + elif c == "," and depth == 0: + args.append(text[start:i]) + start = i + 1 + args.append(text[start:]) + merged: list[str] = [] + for arg in args: + if merged and re.match(r"\s*LAMBDA\b", merged[-1], re.I) and ":" not in merged[-1]: + merged[-1] += "," + arg + else: + merged.append(arg) + return merged + + +def calls(sql: str, name: str, replace: Callable[[list[str]], str]) -> str: + """Rewrite every call of `name`, innermost first, handing its argument list to `replace`.""" + + pattern = re.compile(rf"(? str: + """Polyglot's LATERAL UNNEST of a FLATTEN call, as a subquery with FLATTEN's columns.""" + + out, i = [], 0 + for m in re.finditer(r"LATERAL UNNEST\(\s*input => ", sql): + if m.start() < i: + continue + end = balanced(sql, m.end()) + args = sql[m.end() : end - 1] + outer = re.search(r',\s*"outer" => TRUE', args) is not None + expr = re.sub(r',\s*"outer" => TRUE', "", args) + alias = FLATTEN_COLUMNS.match(sql, end - 1) + if not alias: + continue + name = alias.group(1) + nulls = "CAST(NULL AS BIGINT) AS seq" + if re.search(rf"(? f.k AS value, ' + f"CAST(NULL AS JSON) AS this " + f"FROM (SELECT UNNEST(json_keys({obj})) AS k) AS f) AS {name}" + ) + else: + # An array's elements: INDEX and VALUE per element; OUTER keeps one null row. + items = f"CAST(to_json({expr}) AS JSON[])" + if outer: + items = ( + f"CASE WHEN COALESCE(len({items}), 0) = 0 THEN [CAST(NULL AS JSON)] " + f"ELSE {items} END" + ) + body = ( + f"(SELECT {nulls}, CAST(NULL AS VARCHAR) AS key, CAST(NULL AS VARCHAR) AS path, " + f'CAST(f.i - 1 AS BIGINT) AS "index", f.e AS value, CAST(NULL AS JSON) AS this ' + f"FROM (SELECT UNNEST({items}) AS e, generate_subscripts({items}, 1) AS i) AS f) " + f"AS {name}" + ) + out.append(sql[i : m.start()]) + out.append("LATERAL " + body) + i = alias.end() + out.append(sql[i:]) + return "".join(out) + + +def lambda_arg(arg: str) -> str: + """Snowflake's `x -> body` lambda as DuckDB's `lambda x: body`. Its list holds JSON, so a text + cast of the element unquotes it, as Snowflake's ::VARCHAR does.""" + + m = re.match(r"^\s*(?:LAMBDA\s+)?(\w+)\s*(?:->|:)", arg, re.I) + if m: + element = m.group(1) + arg = re.sub( + rf"CAST\({element} AS (TEXT|VARCHAR)\)", f"json_extract_string({element}, '$')", arg + ) + return re.sub(r"^(\s*)(\w+) -> ", r"\1lambda \2: ", arg) + + +def json_list(arg: str) -> str: + arg = arg.strip() + if arg.startswith(("CAST(to_json(", "list_")): + return arg + return f"CAST(to_json({arg}) AS JSON[])" + + +def casts(sql: str) -> str: + """CAST(x AS VARIANT|OBJECT|ARRAY) makes a JSON value, as Snowflake makes a variant.""" + + i = 0 + pattern = re.compile(r"(? str: + """Snowflake's ASOF JOIN ... MATCH_CONDITION (c) ON k as DuckDB's ASOF JOIN ... ON k AND c.""" + + while m := re.search(r"\s*MATCH_CONDITION \(", sql): + end = balanced(sql, m.end()) + condition = sql[m.end() : end - 1] + rest = sql[end:] + on = re.match(r"(\s*ON [^\n]+)", rest) + if on is None: + raise ValueError("MATCH_CONDITION without an ON clause on the next line") + sql = sql[: m.start()] + on.group(1) + f" AND {condition}" + rest[on.end() :] + return sql + + +def strftime_format(m: re.Match[str]) -> str: + fmt = m.group(2) + if "%" not in fmt: + for old, new in SNOWFLAKE_FORMATS: + fmt = fmt.replace(old, new) + return f"{m.group(1)}'{fmt}')" + + +def json_text(sql: str, model: str | None) -> str: + """CAST(json AS TEXT) keeps a JSON string's quotes in DuckDB; Snowflake's ::VARCHAR drops + them, so a text cast of a JSON value reads it as a string.""" + + pattern = re.compile(r"(? '[^']*'", bare) + or re.fullmatch(r"\w+\.value", bare) + or (head and head.group(1) == "COALESCE" and "to_json(" in bare) + ) + if t and jsonish: + text = f"json_extract_string({value}, '$') AS {t.group(1)}" + sql = sql[: m.end()] + text + sql[end - 1 :] + i = m.end() + return sql + + +def list_call(name: str, target: str) -> Callable[[list[str]], str]: + def replace(args: list[str]) -> str: + if args[0].strip().upper().startswith("WHERE"): + return f"{name}({','.join(args)})" + return f"{target}({', '.join([json_list(args[0]), *map(lambda_arg, args[1:])])})" + + return replace + + +def list_functions(sql: str) -> str: + """Snowflake's array functions on JSON lists, as DuckDB's list functions.""" + + for name, target in LIST_FUNCTIONS.items(): + sql = calls(sql, name, list_call(name, target)) + sql = calls( + sql, "LIST_APPEND", lambda a: f"list_append({json_list(a[0])}, to_json({a[1].strip()}))" + ) + sql = calls( + sql, "LIST_PREPEND", lambda a: f"list_prepend(to_json({a[0].strip()}), {json_list(a[1])})" + ) + sql = calls(sql, "LIST_SORT", lambda a: f"list_sort({', '.join([json_list(a[0]), *a[1:]])})") + sql = calls(sql, "LIST_CONCAT", lambda a: f"list_concat({', '.join(map(json_list, a))})") + return calls( + sql, + "REDUCE", + lambda a: f"list_reduce({json_list(a[0])}, {a[2].strip()}, {a[1].strip()})", + ) + + +def rewrite(sql: str, model: str | None = None) -> str: + """Finish Polyglot's DuckDB translation of one model, function or expression.""" + + sql = flatten(sql) + # A flattened JSON string read as text loses its quotes, as Snowflake's ::VARCHAR does. + sql = re.sub(r"CAST\((\w+)\.value AS (TEXT|VARCHAR)\)", r"(\1.value ->> '$')", sql) + sql = re.sub( + r"(STRFTIME\((?:[^()]|\([^()]*\))*?, )'([^']*)'\)", strftime_format, sql, flags=re.I + ) + sql = re.sub(r"(? '$.\2'", sql, flags=re.I) + sql = re.sub(r"CAST\((JSON\('[^']*'\))\[([\w.]+)\] AS TEXT\)", r"(\1 ->> \2)", sql) + sql = re.sub( + r"(? ('[^']*') AS (TEXT|VARCHAR)\)", r"(\1 ->> \2)", sql) + sql = re.sub( + r"CAST\(sf_strip_null_value\(([\w.]+) -> ('[^']*')\) AS (TEXT|VARCHAR)\)", + r"(\1 ->> \2)", + sql, + ) + sql = json_text(sql, model) + arrow = r"[\w.]+ -> '[^']*'" + return re.sub( + rf"CAST\(COALESCE\(((?:{arrow}, )+{arrow})\) AS (TEXT|VARCHAR)\)", + lambda m: "COALESCE(" + m.group(1).replace(" -> ", " ->> ") + ")", + sql, + ) + + +def replace_cte(sql: str, name: str, body: str) -> str: + m = re.search(rf"(? str: + return replace_cte( + sql, + "generated", + " SELECT CAST(d AS DATE) AS calendar_date\n" + " FROM range(DATE '2025-01-01', DATE '2025-01-01' + INTERVAL 1095 DAY, INTERVAL 1 DAY)" + " AS t(d)", + ) + + +def _category_paths(sql: str) -> str: + tree = ( + " SELECT c.category_id, c.parent_category_id, c.slug, c.display_name, c.sort_order,\n" + " 1 AS depth, '/' || c.slug AS slug_path, c.category_id AS root_category_id\n" + " FROM categories AS c\n WHERE c.is_root\n UNION ALL\n" + " SELECT c.category_id, c.parent_category_id, c.slug, c.display_name, c.sort_order,\n" + " t.depth + 1, t.slug_path || '/' || c.slug, t.root_category_id\n" + " FROM categories AS c\n INNER JOIN tree AS t ON c.parent_category_id = t.category_id" + ) + return re.sub(r"^WITH ", "WITH RECURSIVE ", replace_cte(sql, "tree", tree), count=1) + + +# Snowflake constructs with no mechanical DuckDB form (GENERATOR, CONNECT BY), rewritten by hand. +OVERRIDES: dict[str, Callable[[str], str]] = { + "dim_date": _dim_date, + "int_category_paths": _category_paths, +} + + +def convert(sql: str, model: str | None = None) -> str: + sql = rewrite(sql, model) + return OVERRIDES[model](sql) if model in OVERRIDES else sql + + +def shim_function_name(name: str, arity: int) -> str: + return f"{name}_{arity}" if len(SHIMS[name]) > 1 else name + + +def shim_calls(sql: str) -> str: + """Calls of shims as calls of their project SQL macros.""" + + longest_first = sorted(SHIMS, key=lambda name: len(name), reverse=True) + for name in longest_first: + sql = calls(sql, name, shim_call(name)) + return sql + + +def shim_call(name: str) -> Callable[[list[str]], str]: + def replace(args: list[str]) -> str: + return f"@sql.{shim_function_name(name, len(args))}({','.join(args)})" + + return replace + + +def parameters(body: str, names: list[str]) -> str: + """Each parameter as `@name`, outside string literals and lambda parameters.""" + + parts = re.split(r"('(?:[^']|'')*')", body) + for index in range(0, len(parts), 2): + for name in names: + parts[index] = re.sub(rf"(? pathlib.Path: + """The declaration folder that owns a macro called from `folders`, as SQLBuild requires: + private to one folder, or public at the folders' closest common folder.""" + + if len(folders) == 1: + return folders[0] / "_sqlbuild" / "_macros" / "sql" + common = pathlib.Path(os.path.commonpath(folders)) + if common in (root, root / "models"): + return root / "macros" / "sql" + return common / "_sqlbuild" / "macros" / "sql" + + +def write_shims(root: pathlib.Path) -> None: + """Write each shim overload a model calls, in the folder that owns its callers.""" + + callers: dict[str, set[pathlib.Path]] = {} + for path in root.glob("models/**/*.sql"): + for name in re.findall(r"@sql\.(\w+)\(", path.read_text()): + callers.setdefault(name, set()).add(path.parent) + for name, overloads in SHIMS.items(): + for params, body in overloads: + names = [p.strip() for p in params.split(",")] + macro = shim_function_name(name, len(names)) + if macro not in callers: + continue + directory = macro_folder(root, sorted(callers[macro])) + directory.mkdir(parents=True, exist_ok=True) + returns = RETURNS.get(macro, RETURNS.get(name)) + function = name.removeprefix("sf_").upper() + arguments = ", ".join(f"{p} ANY" for p in names) + (directory / f"{macro}.sql").write_text( + "MACRO (\n" + f" description \"Snowflake's {function} for DuckDB, " + f'taking {len(names)} arguments",\n' + f" arguments ({arguments}),\n" + + (f" returns {returns},\n" if returns else "") + + ");\n\n" + f"{parameters(body, names)}\n" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 3aef15546a..6771ac246c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1281,6 +1281,16 @@ class BenchmarkCoreCopyCompileTestCase: expected_compiled_checksum: str +@dataclass(frozen=True) +class BenchmarkDuckDBPortTestCase: + """A script that ports a Snowflake benchmark project to DuckDB, and the port's build summary.""" + + description: str + port_script: Path + expected_final_line_prefix: str + expected_final_line_fragment: str + + @dataclass(frozen=True) class GeneratedBenchmarkCompileTestCase: """A generated benchmark project, its recorded checksum and its cold compile budget.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py index ed6b2309d4..d6ba97e067 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py @@ -13,6 +13,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( BenchmarkCoreCopyCompileTestCase, + BenchmarkDuckDBPortTestCase, BenchmarkProjectCompileTestCase, BenchmarkProjectTestPlanTestCase, ) @@ -146,3 +147,47 @@ def test_given_benchmark_core_copy_when_compiling_then_leaves_out_the_excluded_a ) assert checksum.returncode == 0, checksum.stdout + checksum.stderr assert checksum.stdout.strip() == test_case.expected_compiled_checksum + + +@pytest.mark.parametrize( + "test_case", + [ + BenchmarkDuckDBPortTestCase( + description="duckshop_duckdb", + port_script=REPO_ROOT / "benchmarks" / "duckshop" / "scripts" / "duckdb_project.py", + expected_final_line_prefix="\u2713 Completed", + expected_final_line_fragment=" FAIL=0 ", + ) + ], + ids=lambda case: case.description, +) +def test_given_duckdb_port_when_building_then_every_model_loads_builds_and_passes( + test_case: BenchmarkDuckDBPortTestCase, tmp_path: Path +) -> None: + """The DuckDB copy is the demonstration project, so it must compile and build end to end.""" + project_dir: Path = tmp_path / test_case.description + ported: subprocess.CompletedProcess[str] = subprocess.run( + ["uv", "run", "-q", "--script", str(test_case.port_script), str(project_dir)], + capture_output=True, + text=True, + check=False, + ) + assert ported.returncode == 0, ported.stdout + ported.stderr + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), + project_dir=project_dir, + ) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + report: dict[str, Any] = json.loads(compiled.stdout) + assert report["summary"]["errors"] == 0, report["diagnostics"] + assert report["summary"]["models"] > 0 + + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + ) + assert built.returncode == 0, built.stdout + built.stderr + final_line: str = built.stdout.strip().splitlines()[-1] + assert final_line.startswith(test_case.expected_final_line_prefix), built.stdout[-4000:] + assert test_case.expected_final_line_fragment in final_line, built.stdout[-4000:] From cd66fcae180b3a05126a7b86bb2da6239d077629 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:07:15 +0000 Subject: [PATCH 2/5] fix(duckshop): drop Snowflake freshness queries from the DuckDB copy A sql freshness query names the Snowflake raw table, which the DuckDB copy does not have, so freshness failed and blocked the models reading that source. --- benchmarks/duckshop/scripts/duckdb_project.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/duckshop/scripts/duckdb_project.py b/benchmarks/duckshop/scripts/duckdb_project.py index 8b02cab5de..0b9e17d92f 100644 --- a/benchmarks/duckshop/scripts/duckdb_project.py +++ b/benchmarks/duckshop/scripts/duckdb_project.py @@ -169,6 +169,9 @@ def write_sources(out: pathlib.Path) -> None: settings["na_values"] = sorted(values) if entry.get("expression"): entry["expression"] = translate(entry["expression"]) + "\n" + if (entry.get("freshness") or {}).get("query"): + # The query reads the Snowflake raw table, which the copy does not have. + del entry["freshness"] path.write_text(yaml.safe_dump(doc, sort_keys=False, width=100)) for path in out.rglob("*.py"): if "functions" not in path.parts: From 53cff26f929eb8010f0fa2571d69a83a0d624966 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:15:59 +0000 Subject: [PATCH 3/5] fix(ui): stop the plan page and lineage graphs jumping while they load Graphs stay hidden until their first fit has painted, so cards no longer appear at unlaid-out positions and slide into place. The plan page keeps one layout while it loads: one overlay covers the result from the first command check until the plan arrives, and the side panel no longer appears late and narrows the page. --- ui/src/lib/lineage/flow-controller.svelte | 28 +++++++++- ui/src/lib/lineage/scope-graph.svelte | 10 +++- ui/src/lib/lineage/scoped-lineage.svelte | 10 +++- ui/src/routes/plan/+page.svelte | 62 +++++++++++++++-------- 4 files changed, 83 insertions(+), 27 deletions(-) diff --git a/ui/src/lib/lineage/flow-controller.svelte b/ui/src/lib/lineage/flow-controller.svelte index 62289b60bf..5e540cba85 100644 --- a/ui/src/lib/lineage/flow-controller.svelte +++ b/ui/src/lib/lineage/flow-controller.svelte @@ -1,21 +1,45 @@ -
+
onselect(picked.filter((n) => n.type === 'asset').map((n) => n.id))} > - + @@ -82,6 +83,11 @@ border-radius: 2px; background: var(--sb-inset); } + .unfitted :global(.svelte-flow__viewport), + .unfitted :global(.svelte-flow__controls) { + /* Not visibility: Svelte Flow sets each measured card visible, which would override it. */ + opacity: 0; + } .graph :global(.svelte-flow__node-asset) { cursor: pointer; } diff --git a/ui/src/lib/lineage/scoped-lineage.svelte b/ui/src/lib/lineage/scoped-lineage.svelte index 719947d621..cc64e4a6ea 100644 --- a/ui/src/lib/lineage/scoped-lineage.svelte +++ b/ui/src/lib/lineage/scoped-lineage.svelte @@ -89,6 +89,7 @@ let nodes = $state.raw([]); let edges = $state.raw([]); let fitFn = $state<((opts?: { duration?: number }) => void) | undefined>(); + let fitted = $state(false); // (1) Rebuild the node/edge set ONLY when the visible neighbourhood changes // (depth steppers / focal). New nodes start at 0,0; dagre places them in (2). @@ -191,7 +192,7 @@
-
+
- + @@ -211,6 +212,11 @@