From eb9e93230e65d8db23e0b453b3a4897701541bb9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:14:48 +0100 Subject: [PATCH] feat(benchmarks): add the DuckShop 10K generator --- .github/workflows/verify.yml | 2 +- Makefile | 8 + benchmarks/duckshop-10k/README.md | 66 + benchmarks/duckshop-10k/checksum.txt | 1 + .../duckshop-10k/duckshop10k/__init__.py | 9 + benchmarks/duckshop-10k/duckshop10k/chains.py | 411 ++++ benchmarks/duckshop-10k/duckshop10k/core.py | 259 +++ .../duckshop-10k/duckshop10k/declarations.py | 374 ++++ benchmarks/duckshop-10k/duckshop10k/domain.py | 624 ++++++ benchmarks/duckshop-10k/duckshop10k/extras.py | 395 ++++ .../duckshop-10k/duckshop10k/intermediate.py | 534 ++++++ benchmarks/duckshop-10k/duckshop10k/layers.py | 550 ++++++ .../duckshop-10k/duckshop10k/manifest.py | 51 + .../duckshop-10k/duckshop10k/mart_layer.py | 532 ++++++ benchmarks/duckshop-10k/duckshop10k/marts.py | 456 +++++ .../duckshop-10k/duckshop10k/project.py | 216 +++ .../duckshop-10k/duckshop10k/rules_gen.py | 140 ++ benchmarks/duckshop-10k/duckshop10k/specs.py | 1681 +++++++++++++++++ .../duckshop-10k/duckshop10k/staging.py | 738 ++++++++ benchmarks/duckshop-10k/duckshop10k/stats.py | 70 + .../duckshop-10k/duckshop10k/tests_gen.py | 664 +++++++ benchmarks/duckshop-10k/duckshop10k/world.py | 508 +++++ benchmarks/duckshop-10k/generate.py | 65 + .../duckshop-10k/tests/test_generator.py | 160 ++ .../cli/commands/main/compile/_test_types.py | 11 + .../cli/commands/main/compile/helpers.py | 14 + .../main/compile/test_duckshop_10k_compile.py | 78 + 27 files changed, 8616 insertions(+), 1 deletion(-) create mode 100644 benchmarks/duckshop-10k/README.md create mode 100644 benchmarks/duckshop-10k/checksum.txt create mode 100644 benchmarks/duckshop-10k/duckshop10k/__init__.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/chains.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/core.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/declarations.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/domain.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/extras.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/intermediate.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/layers.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/manifest.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/mart_layer.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/marts.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/project.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/rules_gen.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/specs.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/staging.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/stats.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/tests_gen.py create mode 100644 benchmarks/duckshop-10k/duckshop10k/world.py create mode 100644 benchmarks/duckshop-10k/generate.py create mode 100644 benchmarks/duckshop-10k/tests/test_generator.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 96aeac78c6..3f51d3fbb9 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -163,7 +163,7 @@ jobs: performance=true rules_performance=true ;; - src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*) + benchmarks/duckshop-10k/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*) broad=true performance=true ;; diff --git a/Makefile b/Makefile index 0dc9b83fe5..54dfc049b2 100644 --- a/Makefile +++ b/Makefile @@ -299,6 +299,14 @@ test-e2e-dense-compile-performance: -n auto --dist loadfile \ -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \ -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes +# DuckShop 10K runs after the dense guards, alone, so it never shares the memory limit with them. + if [ "$(SQLBUILD_BENCHMARK_MODELS)" = "10000" ]; then \ + env PYTHONUNBUFFERED=1 uv run pytest benchmarks/duckshop-10k/tests -q --color=yes && \ + env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ + tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py \ + -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)_" \ + -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes; \ + fi test-e2e-cache-compile-performance: test -n "$(SQLBUILD_BENCHMARK_MODELS)" diff --git a/benchmarks/duckshop-10k/README.md b/benchmarks/duckshop-10k/README.md new file mode 100644 index 0000000000..c90c334e42 --- /dev/null +++ b/benchmarks/duckshop-10k/README.md @@ -0,0 +1,66 @@ +# DuckShop 10K + +A deterministic generator for a ten-thousand-model SQLBuild project in Snowflake SQL. It describes +Mallard & Co., the fictional duck-themed home goods business of the +[DuckShop](../duckshop/README.md) project, after a decade of growth: 40 storefronts in 10 regions, +marketplace partners, carriers, warehouses, retail stores, suppliers, payment gateways, ad +networks, messaging, affiliates, support tools, ledgers, wholesale desks, returns portals and a +product catalogue. Every name and value is invented. + +Only the generator, its tests and `checksum.txt` are committed. The generated project is written +to a directory you choose. + +## Usage + +```bash +python benchmarks/duckshop-10k/generate.py --out /path/to/duckshop-10k # write the project +python benchmarks/duckshop-10k/generate.py --verify /path/to/duckshop-10k # check a written copy +python benchmarks/duckshop-10k/generate.py --check # check an in-memory build +cd /path/to/duckshop-10k && sqb compile # no warehouse connection needed +``` + +`--profile smoke` writes a small project of about 1,100 models with the same templates, for fast +iteration. `checksum.txt` records the full profile only. + +## Determinism and versioning + +The generator uses no randomness, clocks or environment: the same generator version always writes +byte-identical files. `checksum.txt` holds the generator version, the file count and the SHA-256 of +`MANIFEST.sha256`, a sorted list of the SHA-256 of every generated file. + +When a change alters any generated file, bump `GENERATOR_VERSION` in `duckshop10k/__init__.py` and +run `python benchmarks/duckshop-10k/generate.py --update-checksum`. + +## Shape + +| | Count | +|---|---| +| Models | 10,282 | +| Staging, intermediate, marts and snapshots | 6,149 (59.8%), 1,961 (19.1%), 2,162 (21.0%), 10 | +| Sources | 2,466 table sources and one expression source | +| Unit tests and scenarios | 288 and 2 | +| Dependency depth | 11 at most; 168 models at depth 7 or more | + +Most models read one to four upstreams; a few dozen wide marts, unions and scorecards read ten or +more. Staging deduplicates change logs with a change-log macro, types three platform layouts +(legacy columns with local times, typed columns with offset times, and JSON documents with epoch +times) and flattens arrays into child models. The intermediate layer converts every instance to a +shared shape (UTC times, major-unit amounts, mapped statuses), enriches headers with child totals, +unions instances by region and company, converts currencies with `ASOF JOIN`, sessionises +clickstream events in microbatches and builds customer identity, value and segment models and a +budget variance chain. Marts give every instance, team and region its reports, and publish facts and +dimensions with enforced contracts. + +The models that use `HASH`, `TIMESTAMP_LTZ` or `CURRENT_TIMESTAMP()` are tagged `known_divergence`, +because their results depend on the engine or the session. + +## Checks + +- `benchmarks/duckshop-10k/tests/` checks the shape, the determinism of two builds, the recorded + checksum and the generated custom Rule's tests, offline. +- `tests/e2e/.../compile/test_duckshop_10k_compile.py` generates the project, verifies it against + `checksum.txt` and compiles it cold with the checkout's build, requiring zero errors and zero + warnings. + +CI runs both in the 10,000-model cell of the dense compile job, under its 4 GiB memory limit, after +the dense guards have finished, so the DuckShop compile never shares the limit with them. diff --git a/benchmarks/duckshop-10k/checksum.txt b/benchmarks/duckshop-10k/checksum.txt new file mode 100644 index 0000000000..3a8ae1527c --- /dev/null +++ b/benchmarks/duckshop-10k/checksum.txt @@ -0,0 +1 @@ +generator 1 files 10874 sha256 4d276ad2428841efa2a1b851c83863bea70505d9e085a71390401399b5667667 diff --git a/benchmarks/duckshop-10k/duckshop10k/__init__.py b/benchmarks/duckshop-10k/duckshop10k/__init__.py new file mode 100644 index 0000000000..4a2841094a --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/__init__.py @@ -0,0 +1,9 @@ +"""Deterministic generator for DuckShop 10K, a ten-thousand-model SQLBuild project. + +The same generator version always writes byte-identical output. Bump GENERATOR_VERSION, and +regenerate checksum.txt, whenever a change alters any generated file. +""" + +GENERATOR_VERSION = "1" +SEED = 20261024 +PROJECT_NAME = "duckshop_10k" diff --git a/benchmarks/duckshop-10k/duckshop10k/chains.py b/benchmarks/duckshop-10k/duckshop10k/chains.py new file mode 100644 index 0000000000..fc7257b679 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/chains.py @@ -0,0 +1,411 @@ +"""The long chains: customer value and segments, the finance close, scorecards and board packs.""" + +from __future__ import annotations + +from duckshop10k import marts +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.intermediate import to_usd +from duckshop10k.layers import Intermediate +from duckshop10k.staging import with_query +from duckshop10k.world import REGIONS + + +def _model( + project: Project, + name: str, + folder: str, + layer: str, + description: str, + columns: list[Column], + ctes: list[str], + refs: list[str], + **extra: object, +) -> Model: + return project.add( + Model( + name=name, + folder=folder, + layer=layer, + description=description, + columns=columns, + body=with_query(ctes), + refs=refs, + **extra, # type: ignore[arg-type] + ) + ) + + +def customer_chain(project: Project, result: Intermediate) -> None: + orders = result.combined.get("orders_usd") + identity = result.combined.get("customer_identity") + if orders is None or identity is None: + return + folder = "models/intermediate/customers" + value = _model( + project, + "int_customer_value", + folder, + "intermediate", + "One row per global customer with order count, USD revenue and recency across every " + "storefront account linked to them", + [ + Column("global_customer_key", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("linked_account_count", "INT"), + Column("order_count", "INT", ("non_negative",)), + Column("gross_revenue_usd", "NUMBER(18, 2)"), + Column("first_order_at_utc", "TIMESTAMP_NTZ"), + Column("last_order_at_utc", "TIMESTAMP_NTZ"), + Column("days_since_last_order", "INT"), + ], + [ + import_cte("orders", f'__ref("{orders.name}")'), + import_cte("identities", f'__ref("{identity.name}")'), + cte( + "linked", + "SELECT\n" + + select_list( + [ + "i.global_customer_key", + "i.linked_account_count", + "o.order_id", + "o.placed_at_utc", + "o.gross_amount_usd", + ] + ) + + "\nFROM orders AS o\nINNER JOIN identities AS i\n" + " ON o.storefront_code = i.storefront_code\n AND o.customer_id = i.customer_id", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "l.global_customer_key", + "MAX(l.linked_account_count)::INT AS linked_account_count", + "COUNT(DISTINCT l.order_id)::INT AS order_count", + "SUM(l.gross_amount_usd)::NUMBER(18, 2) AS gross_revenue_usd", + "MIN(l.placed_at_utc) AS first_order_at_utc", + "MAX(l.placed_at_utc) AS last_order_at_utc", + "DATEDIFF('day', MAX(l.placed_at_utc), '@@valuation_date'::DATE)::INT" + " AS days_since_last_order", + ] + ) + + "\nFROM linked AS l\nGROUP BY\n l.global_customer_key", + ), + ], + [orders.name, identity.name], + ) + segments = _model( + project, + "int_customer_segments", + folder, + "intermediate", + "One row per global customer scored on recency, frequency and monetary value in quintiles, " + "with the segment those scores place them in", + [ + Column("global_customer_key", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("order_count", "INT"), + Column("gross_revenue_usd", "NUMBER(18, 2)"), + Column("recency_score", "INT"), + Column("frequency_score", "INT"), + Column("monetary_score", "INT"), + Column("segment", "VARCHAR", ("not_null",)), + Column("is_vip", "BOOLEAN"), + ], + [ + import_cte("customer_value", f'__ref("{value.name}")'), + cte( + "scored", + "SELECT\n" + + select_list( + [ + "v.global_customer_key", + "v.order_count", + "v.gross_revenue_usd", + "NTILE(5) OVER (ORDER BY v.days_since_last_order DESC, " + "v.global_customer_key) AS recency_score", + "NTILE(5) OVER (ORDER BY v.order_count, v.global_customer_key)" + " AS frequency_score", + "NTILE(5) OVER (ORDER BY v.gross_revenue_usd, v.global_customer_key)" + " AS monetary_score", + ] + ) + + "\nFROM customer_value AS v", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "s.global_customer_key", + "s.order_count", + "s.gross_revenue_usd", + "s.recency_score::INT AS recency_score", + "s.frequency_score::INT AS frequency_score", + "s.monetary_score::INT AS monetary_score", + "CASE\n WHEN s.recency_score >= 4 AND s.frequency_score >= 4 " + "THEN 'champion'" + "\n WHEN s.recency_score >= 4 THEN 'recent'\n" + " WHEN s.frequency_score >= 4 THEN 'loyal_at_risk'\n" + " WHEN s.recency_score <= 2 AND s.frequency_score <= 2 THEN 'lapsed'\n" + " ELSE 'developing'\n END::VARCHAR AS segment", + '(s.gross_revenue_usd >= @const("vip_ltv_threshold"))::BOOLEAN AS is_vip', + ] + ) + + "\nFROM scored AS s", + ), + ], + [value.name], + ) + dim = marts.publish( + project, + "dim_customers", + "models/marts/core", + segments, + "global_customer_key", + "One row per global customer with their value scores and segment", + ) + segment_mix = marts.share_by( + project, + "rpt_marketing__segment_mix", + "models/marts/marketing", + dim, + "segment", + "Global customers", + ) + vip_mix = marts.share_by( + project, + "rpt_marketing__vip_mix", + "models/marts/marketing", + dim, + "is_vip", + "Global customers", + ) + result.combined["dim_customers"] = dim + if segment_mix is not None and vip_mix is not None: + result.combined["segment_mix"] = segment_mix + + +def finance_chain(project: Project, result: Intermediate) -> None: + entries = result.combined.get("ledgers__gl_entries") + budgets = result.combined.get("ledgers__budgets") + if entries is None or budgets is None: + return + folder = "models/intermediate/ledgers" + usd = to_usd( + project, + "int_gl_entries_usd", + folder, + entries, + ["gl_entry_id"], + "posted_date_local", + "Every general ledger entry of every legal entity converted to USD at the reference rate " + "of its posting day", + ) + monthly = _model( + project, + "int_gl_monthly_usd", + folder, + "intermediate", + "Posted value in USD per legal entity, cost centre and month", + [ + Column("legal_entity_code", "VARCHAR", ("not_null",)), + Column("cost_centre", "VARCHAR"), + Column("period_month", "DATE", ("not_null",)), + Column("entry_count", "INT"), + Column("posted_value_usd", "NUMBER(18, 2)"), + ], + [ + import_cte("entries", f'__ref("{usd.name}")'), + cte( + "final", + "SELECT\n" + + select_list( + [ + "e.legal_entity_code", + "e.cost_centre", + "DATE_TRUNC('month', e.posted_date_local)::DATE AS period_month", + "COUNT(*)::INT AS entry_count", + "SUM(e.posted_value_amount_usd)::NUMBER(18, 2) AS posted_value_usd", + ] + ) + + "\nFROM entries AS e\nGROUP BY\n e.legal_entity_code,\n e.cost_centre,\n" + " DATE_TRUNC('month', e.posted_date_local)::DATE", + ), + ], + [usd.name], + ) + variance = _model( + project, + "int_gl_budget_variance", + folder, + "intermediate", + "Actual against budgeted spend per legal entity, cost centre and month, with the variance " + "as an amount and a percentage of budget", + [ + Column("variance_key", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("legal_entity_code", "VARCHAR"), + Column("cost_centre", "VARCHAR"), + Column("period_month", "DATE"), + Column("actual_usd", "NUMBER(18, 2)"), + Column("budget_amount", "NUMBER(18, 2)"), + Column("variance_amount", "NUMBER(18, 2)"), + Column("variance_percent", "NUMBER(9, 4)"), + ], + [ + import_cte("actuals", f'__ref("{monthly.name}")'), + import_cte("budget_lines", f'__ref("{budgets.name}")'), + cte( + "budget_months", + "SELECT\n" + + select_list( + [ + "b.legal_entity_code", + "b.cost_centre", + "DATE_TRUNC('month', b.period_date)::DATE AS period_month", + "SUM(b.budgeted_amount) AS budget_amount", + ] + ) + + "\nFROM budget_lines AS b\nGROUP BY\n b.legal_entity_code,\n b.cost_centre,\n" + " DATE_TRUNC('month', b.period_date)::DATE", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "(a.legal_entity_code || ':' || COALESCE(a.cost_centre, '-') || ':' || " + "TO_CHAR(a.period_month, 'YYYY-MM'))::VARCHAR AS variance_key", + "a.legal_entity_code", + "a.cost_centre", + "a.period_month", + "a.posted_value_usd AS actual_usd", + "COALESCE(m.budget_amount, 0)::NUMBER(18, 2) AS budget_amount", + "(a.posted_value_usd - COALESCE(m.budget_amount, 0))::NUMBER(18, 2)" + " AS variance_amount", + "@sql.percent_of(a.posted_value_usd - m.budget_amount, m.budget_amount)" + " AS variance_percent", + ] + ) + + "\nFROM actuals AS a\nLEFT JOIN budget_months AS m\n" + " ON a.legal_entity_code = m.legal_entity_code\n" + " AND a.cost_centre = m.cost_centre\n AND a.period_month = m.period_month", + ), + ], + [monthly.name, budgets.name], + ) + fact = marts.publish( + project, + "fct_gl_budget_variance", + "models/marts/core", + variance, + "variance_key", + "One row per legal entity, cost centre and month with actual and budgeted spend", + ) + by_entity = marts.share_by( + project, + "rpt_finance__variance_by_entity", + "models/marts/finance", + fact, + "legal_entity_code", + "Budget variance", + ) + quarterly = marts.period_rollup( + project, + "rpt_finance__variance_quarterly", + "models/marts/finance", + fact, + "quarter", + "Budget variance", + ) + monthly_report = marts.period_rollup( + project, + "rpt_finance__variance_monthly", + "models/marts/finance", + fact, + "month", + "Budget variance", + ) + for key, model in ( + ("variance_by_entity", by_entity), + ("variance_quarterly", quarterly), + ("variance_monthly", monthly_report), + ): + if model is not None: + result.combined[key] = model + + +def scorecards(project: Project, result: Intermediate) -> None: + """Wide per-storefront and per-team scorecards, then executive packs over the scorecards.""" + shop_cards: list[Model] = [] + for code in sorted(project.models): + if not (code.startswith("rpt_shop_") and code.endswith("__sales_weekly")): + continue + shop = code.removeprefix("rpt_").removesuffix("__sales_weekly") + parts = [ + ("sales", f"rpt_{shop}__sales_weekly"), + ("payments", f"rpt_{shop}__payments_weekly"), + ("refunds", f"rpt_{shop}__refunds_weekly"), + ] + sources = [(alias, project.models[name]) for alias, name in parts if name in project.models] + if len(sources) < 2: + continue + shop_cards.append( + marts.wide_daily( + project, + f"rpt_{shop}__weekly_scorecard", + f"models/marts/storefronts/{shop}", + sources, + f"One row per week with the sales, payments and refunds of {shop} side by side", + ) + ) + team_cards: list[Model] = [] + for team in ("finance", "marketing"): + for region in REGIONS: + prefix = f"rpt_{team}_{region.code}__" + weekly = [ + (name.removeprefix(prefix).removesuffix("_weekly"), project.models[name]) + for name in sorted(project.models) + if name.startswith(prefix) and name.endswith("_weekly") + ] + if len(weekly) < 2: + continue + team_cards.append( + marts.wide_daily( + project, + f"rpt_{team}_{region.code}__scorecard", + f"models/marts/{team}/{region.code}", + weekly, + f"One row per week with every weekly {team} headline for {region.name} " + "side by side", + ) + ) + folder = "models/marts/executive" + for team in ("finance", "marketing"): + cards = [card for card in team_cards if card.name.startswith(f"rpt_{team}_")] + if len(cards) >= 2: + pack = marts.wide_daily( + project, + f"rpt_executive__{team}_regions", + folder, + [(card.name.split("_", 2)[2].split("__")[0], card) for card in cards], + f"One row per period with the first {team} headline of every region side by side", + ) + result.combined[f"executive_{team}"] = pack + board = [ + (alias, result.combined[key]) + for alias, key in ( + ("finance", "executive_finance"), + ("marketing", "executive_marketing"), + ("variance", "variance_monthly"), + ) + if key in result.combined + ] + if len(board) >= 2: + marts.wide_daily( + project, + "rpt_executive__board_pack", + folder, + board, + "One row per period with the board's headline finance, marketing and budget figures", + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/core.py b/benchmarks/duckshop-10k/duckshop10k/core.py new file mode 100644 index 0000000000..56275eac87 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/core.py @@ -0,0 +1,259 @@ +"""Models, declarations and the project registry every layer generator writes into.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath + + +@dataclass(frozen=True) +class Column: + name: str + type: str + audits: tuple[str, ...] = () + nullable: bool = True + + def render(self) -> str: + parts = [f"type {self.type}"] + if not self.nullable: + parts.append("nullable false") + if self.audits: + parts.append(f"audits [{', '.join(self.audits)}]") + return f" {self.name} ({', '.join(parts)})," + + +@dataclass +class Model: + name: str + folder: str + layer: str + description: str + columns: list[Column] + body: str + config: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + refs: list[str] = field(default_factory=list) + sources: list[str] = field(default_factory=list) + + @property + def path(self) -> str: + return f"{self.folder}/{self.name}.sql" + + def column(self, name: str) -> Column: + return next(column for column in self.columns if column.name == name) + + def has(self, name: str) -> bool: + return any(column.name == name for column in self.columns) + + def render(self) -> str: + lines = ["MODEL ("] + lines.extend(_description_lines(self.description)) + if self.tags: + lines.append(f" tags [{', '.join(self.tags)}],") + lines.extend( + f" {line}," if not line.endswith(",") else f" {line}" for line in self.config + ) + lines.append(" columns (") + lines.extend(column.render() for column in self.columns) + lines.append(" ),") + lines.append(");") + return "\n".join(lines) + "\n\n" + self.body.strip() + "\n" + + +def _description_lines(text: str) -> list[str]: + if len(text) <= 80: + return [f' description "{text}",'] + words = text.split() + wrapped: list[str] = [] + line = "" + for word in words: + if line and len(line) + 1 + len(word) > 96: + wrapped.append(line) + line = word + else: + line = f"{line} {word}" if line else word + wrapped.append(line) + return [' description """', *(f" {item}" for item in wrapped), ' """,'] + + +@dataclass(frozen=True) +class Declaration: + """A reusable declaration whose folder is decided by where it is used.""" + + name: str + role: str # macros/sql, macros/python, constants, enums, audits/generic, schemas + content: str + file_name: str = "" + + @property + def filename(self) -> str: + if self.file_name: + return self.file_name + suffix = ".py" if self.role == "macros/python" else ".sql" + return f"{self.name}{suffix}" + + +_SQL_MACRO = re.compile(r"@sql\.([a-z_0-9]+)\(") +_PY_MACRO = re.compile(r"@py\.([a-z_0-9]+)\(") +_CONST = re.compile(r'@const\("([a-z_0-9]+)"\)') +_ENUM = re.compile(r'@enum\("([a-z_0-9]+)"\)') +_AUDIT = re.compile(r"\baudits \[([^\]]*)\]") +_AUDIT_NAME = re.compile(r"(?:^|,)\s*([a-z_0-9]+)") + + +def declaration_uses(text: str) -> set[tuple[str, str]]: + """(kind, name) of every macro, constant, enum and generic audit a file uses by name.""" + uses: set[tuple[str, str]] = set() + uses.update(("macro", name) for name in _SQL_MACRO.findall(text)) + uses.update(("macro", name) for name in _PY_MACRO.findall(text)) + uses.update(("constant", name) for name in _CONST.findall(text)) + uses.update(("enum", name) for name in _ENUM.findall(text)) + for group in _AUDIT.findall(text): + for name in _AUDIT_NAME.findall(re.sub(r"\([^)]*\)", "", group)): + uses.add(("audit", name)) + return uses + + +_KIND_OF_ROLE = { + "macros/sql": "macro", + "macros/python": "macro", + "constants": "constant", + "enums": "enum", + "audits/generic": "audit", + "schemas": "schema", +} + +BUILT_IN_AUDITS = frozenset({"not_null", "unique", "accepted_values", "relationships"}) + + +class Project: + def __init__(self) -> None: + self.models: dict[str, Model] = {} + self.files: dict[str, str] = {} + self.declarations: dict[tuple[str, str], Declaration] = {} + # (rule, declaration key or model path, reason), rendered into sqlbuild_project.toml + self.exceptions: list[tuple[str, tuple[str, str] | str, str]] = [] + + def add(self, model: Model) -> Model: + if model.name in self.models: + raise ValueError(f"duplicate model {model.name}") + self.models[model.name] = model + return model + + def ref(self, name: str) -> Model: + return self.models[name] + + def write(self, path: str, content: str) -> None: + if path in self.files: + raise ValueError(f"duplicate file {path}") + self.files[path] = content + + def declare(self, declaration: Declaration) -> None: + key = (_KIND_OF_ROLE[declaration.role], declaration.name) + if key in self.declarations: + raise ValueError(f"duplicate declaration {key}") + self.declarations[key] = declaration + + def render(self) -> dict[str, str]: + """Every file of the project, with each declaration placed beside its users.""" + files = dict(self.files) + for model in self.models.values(): + files[model.path] = model.render() + consumers = self._consumers(files) + placed: dict[tuple[str, str], str] = {} + for key, declaration in sorted(self.declarations.items()): + folders = consumers.get(key) + if not folders: + continue + path = f"{_placement(folders, declaration.role)}/{declaration.filename}" + placed[key] = path + files[path] = declaration.content + blocks: list[str] = [] + for rule, subject, reason in self.exceptions: + path = placed.get(subject) if isinstance(subject, tuple) else subject + if path is None: + continue + blocks.append( + f'[[rules.rule_exceptions]]\nrule = "{rule}"\npath = "{path}"\n' + f'reason = """\n{reason.strip()}\n"""\n' + ) + if blocks: + files["sqlbuild_project.toml"] += "\n" + "\n".join(blocks) + return dict(sorted(files.items())) + + def _consumers(self, files: dict[str, str]) -> dict[tuple[str, str], set[str]]: + direct: dict[tuple[str, str], set[str]] = {} + for path, text in files.items(): + if not path.endswith(".sql") or not path.startswith(("models/", "tests/")): + continue + folder = str(PurePosixPath(path).parent) + uses = declaration_uses(text) + if "audit_factories [fact_table_quality]" in text: + uses.add(("audit", "has_rows")) + for use in uses: + if use[0] == "audit" and use[1] in BUILT_IN_AUDITS: + continue + direct.setdefault(use, set()).add(folder) + for path, text in files.items(): + if path.startswith(("sources/", "seeds/")): + for use in declaration_uses(text): + direct.setdefault(use, set()).add(path.split("/")[0]) + # A declaration used inside another declaration is used wherever that one is. + uses_of: dict[tuple[str, str], set[tuple[str, str]]] = { + key: declaration_uses(declaration.content) & set(self.declarations) + for key, declaration in self.declarations.items() + } + resolved: dict[tuple[str, str], set[str]] = {} + + def resolve(key: tuple[str, str], stack: tuple[tuple[str, str], ...]) -> set[str]: + if key in resolved: + return resolved[key] + folders = set(direct.get(key, set())) + for other, used in uses_of.items(): + if key in used and other not in stack: + folders |= resolve(other, (*stack, key)) + resolved[key] = folders + return folders + + for key in self.declarations: + resolve(key, ()) + return resolved + + +def _placement(folders: set[str], role: str) -> str: + roots = {folder.split("/")[0] for folder in folders} + if len(roots) > 1: + return role + parts = [PurePosixPath(folder).parts for folder in folders] + common: list[str] = [] + for level in zip(*parts, strict=False): + if len(set(level)) != 1: + break + common.append(level[0]) + if len(common) < 2: + return role + owner = "/".join(common) + private = all(folder == owner for folder in folders) + return f"{owner}/_sqlbuild/{'_' if private else ''}{role}" + + +def cte(name: str, body: str) -> str: + return f"{name} AS (\n{_indent(body.strip())}\n)" + + +def _indent(text: str, spaces: int = 2) -> str: + pad = " " * spaces + return "\n".join(f"{pad}{line}" if line else line for line in text.splitlines()) + + +def select_list(items: list[str]) -> str: + return ",\n".join(f" {item}" for item in items) + + +def query(ctes: list[str], final: str = "final") -> str: + return "WITH " + ",\n\n".join(ctes) + f"\n\nSELECT *\nFROM {final}\n" + + +def import_cte(name: str, relation: str) -> str: + return cte(name, f"SELECT *\nFROM {relation}") diff --git a/benchmarks/duckshop-10k/duckshop10k/declarations.py b/benchmarks/duckshop-10k/duckshop10k/declarations.py new file mode 100644 index 0000000000..5045c3c252 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/declarations.py @@ -0,0 +1,374 @@ +"""Macros, enums, constants, generic audits, schemas and functions shared across the project.""" + +from __future__ import annotations + +from duckshop10k.core import Declaration, Project +from duckshop10k.world import CURRENCY_EXPONENT + +SQL_MACROS: dict[str, str] = { + "cdc_latest": """MACRO ( + description "Latest surviving version of each key in an append-only change log", + arguments ( + src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ), + key COLUMN OF src, + ), +); + +WITH cdc_versions AS ( + SELECT * + FROM @src + QUALIFY ROW_NUMBER() OVER (PARTITION BY @key ORDER BY _seq DESC) = 1 +), + +cdc_latest AS ( + SELECT * + FROM cdc_versions + WHERE _op <> @enum("cdc_op").DELETE +) +""", + "cdc_history": """MACRO ( + description "Every version of each key in a change log, keeping the last one of each load", + arguments ( + src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ), + key COLUMN OF src, + ), +); + +WITH cdc_history AS ( + SELECT * + FROM @src + QUALIFY ROW_NUMBER() OVER (PARTITION BY @key, _loaded_at ORDER BY _seq DESC) = 1 +) +""", + "load_metadata": """MACRO ( + description "Standard lineage columns carried from a change-log row", + arguments (src RELATION (_seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ)), +); + +SELECT + @src._seq AS source_seq, + TO_TIMESTAMP_NTZ(@src._loaded_at) AS source_loaded_at +""", + "normalise_email": """MACRO ( + description "Lower-cased, trimmed email address, or NULL when it is not a plausible address", + arguments (email VARCHAR), + returns VARCHAR, +); + +IFF( + REGEXP_LIKE(LOWER(TRIM(@email)), @const("email_pattern")), + REGEXP_REPLACE(LOWER(TRIM(@email)), '\\\\+[^@]*@', '@'), + NULL +) +""", + "normalise_phone": """MACRO ( + description "Digits-only phone number with a leading plus, or NULL when too short", + arguments (phone VARCHAR), + returns VARCHAR, +); + +IFF(LENGTH(REGEXP_REPLACE(@phone, '[^0-9]', '')) >= 7, '+' +|| LTRIM( + REGEXP_REPLACE(@phone, '[^0-9]', ''), + '0' +), NULL) +""", + "normalise_sku": """MACRO ( + description "Canonical SKU: upper case with whitespace removed and underscores as hyphens", + arguments (sku VARCHAR), + returns VARCHAR, +); + +UPPER(TRANSLATE( + REGEXP_REPLACE(TRIM(@sku), '\\\\s+', ''), + '_', + '-' +)) +""", + "json_text": """MACRO ( + description "Text of a semi-structured value, treating JSON null and blank strings as SQL NULL", + arguments (value VARIANT), + returns VARCHAR, +); + +NULLIF( + TRIM(STRIP_NULL_VALUE(@value)::VARCHAR), + '' +) +""", + "epoch_ms_to_utc": """MACRO ( + description "UTC timestamp from epoch milliseconds", + arguments (epoch_ms NUMBER(38, 0)), + returns TIMESTAMP_NTZ, +); + +TO_TIMESTAMP_NTZ(@epoch_ms, 3) +""", + "safe_divide": """MACRO ( + description "Division that returns NULL instead of failing on a zero denominator", + arguments (numerator NUMBER(38, 6), denominator NUMBER(38, 6)), + returns NUMBER(38, 6), +); + +( + @numerator / NULLIF(@denominator, 0) +)::NUMBER(38, 6) +""", + "money_from_minor": """MACRO ( + description "Converts an integer amount in minor units to major units for a currency exponent", + arguments (amount NUMBER(38, 0), exponent NUMBER(38, 0), places INT CONST DEFAULT 2), + returns NUMBER(38, 6), +); + +ROUND(@sql.safe_divide(@amount, POWER(10, @exponent)::NUMBER(38, 0)), @places)::NUMBER(38, 6) +""", + "local_business_date": """MACRO ( + description "Calendar date of a UTC timestamp in an IANA time zone", + arguments (ts_utc TIMESTAMP_NTZ, tz VARCHAR), + returns DATE, +); + +CONVERT_TIMEZONE('UTC', @tz, @ts_utc)::DATE +""", + "local_to_utc": """MACRO ( + description "UTC timestamp of a wall-clock time recorded in an IANA time zone", + arguments (ts_local TIMESTAMP_NTZ, tz VARCHAR), + returns TIMESTAMP_NTZ, +); + +CONVERT_TIMEZONE(@tz, 'UTC', @ts_local)::TIMESTAMP_NTZ +""", + "tz_to_utc": """MACRO ( + description "UTC wall-clock time of an offset timestamp", + arguments (ts TIMESTAMP_TZ), + returns TIMESTAMP_NTZ, +); + +CONVERT_TIMEZONE('UTC', @ts)::TIMESTAMP_NTZ +""", + "status_bucket": """MACRO ( + description "Order status bucket for an order status, used by every sales report", + arguments (status ENUM order_status), + returns VARCHAR, +); + +CASE + WHEN @status IN (@enum("order_status").CANCELLED, @enum("order_status").PENDING) + THEN 'open_or_void' + WHEN @status = @enum("order_status").RETURNED + THEN 'returned' + ELSE 'fulfilled' +END +""", + "percent_of": """MACRO ( + description "Share of a part in a whole as a percentage, NULL when the whole is zero", + arguments (part NUMBER(38, 6), whole NUMBER(38, 6), places INT CONST DEFAULT 2), + returns NUMBER(9, 4), +); + +ROUND(100 * @sql.safe_divide(@part, @whole), @places)::NUMBER(9, 4) +""", + "week_start": """MACRO ( + description "Monday of the ISO week a date falls in", + arguments (day DATE), + returns DATE, +); + +DATE_TRUNC('week', @day)::DATE +""", +} + +PY_MACROS: dict[str, tuple[str, str]] = { + "union_relations": ( + "relations.py", + '''"""Set operations over several relations with the same named columns.""" + +from sqlbuild.refs import SqlResourceRef + + +def union_relations(relations: list[SqlResourceRef], columns: list[str]) -> str: + """UNION ALL of the same named columns from each relation, tagged with its relation name.""" + projection = ", ".join(columns) + return "\\nUNION ALL\\n".join( + f"SELECT CAST('{relation.name}' AS VARCHAR) AS source_relation, {projection} " + f"FROM {relation}" + for relation in relations + ) +''', + ), +} + +ENUMS: dict[str, str] = { + "cdc_op": """ENUM ( + name cdc_op, + members ( + INSERT "I", + UPDATE "U", + DELETE "D", + ), +); +""", + "order_status": """ENUM ( + name order_status, + members [PENDING, PAID, PACKED, SHIPPED, DELIVERED, CANCELLED, RETURNED], +); +""", + "sales_channel": """ENUM ( + name sales_channel, + members ( + WEB "web", + APP "app", + STORE "store", + MARKETPLACE "marketplace", + WHOLESALE "wholesale", + PHONE "phone", + ), +); +""", + "ticket_priority": """ENUM ( + name ticket_priority, + members (URGENT 1, HIGH 2, NORMAL 3, LOW 4), +); +""", +} + +CONSTANTS: dict[str, str] = { + "email_pattern": ( + 'CONSTANT (name email_pattern, value "^[a-z0-9._%+-]+@[a-z0-9.-]+\\\\.[a-z]{2,}$");\n' + ), + "bot_user_agent_pattern": ( + 'CONSTANT (name bot_user_agent_pattern, value ".*(bot|crawler|spider|headless).*");\n' + ), + "session_gap_minutes": "CONSTANT (name session_gap_minutes, value 30);\n", + "vip_ltv_threshold": ('CONSTANT (name vip_ltv_threshold, type decimal, value "1500.00");\n'), + "history_start_date": 'CONSTANT (name history_start_date, value "2025-01-01");\n', +} + +AUDITS: dict[str, str] = { + "valid_currency": """AUDIT (); + +WITH currencies AS ( + SELECT * + FROM __seed("currencies") +), + +final AS ( + SELECT r.* + FROM @relation AS r + LEFT JOIN currencies AS c + ON r.@column = c.currency_code + WHERE + r.@column IS NOT NULL + AND c.currency_code IS NULL +) + +SELECT * +FROM final +""", + "non_negative": """AUDIT (); + +WITH final AS ( + SELECT * + FROM @relation + WHERE @column < 0 +) + +SELECT * +FROM final +""", + "has_rows": """AUDIT (); + +WITH counted AS ( + SELECT COUNT(*) AS row_count + FROM @relation +), + +final AS ( + SELECT c.row_count + FROM counted AS c + WHERE c.row_count < @minimum_rows +) + +SELECT * +FROM final +""", + "match_rate": """AUDIT ( + evaluation measurement, + value match_rate, + sample_count total_rows, + sample_unit rows, +); + +MEASURE ( + WITH final AS ( + SELECT + COUNT(*) AS total_rows, + 100.0 * AVG(IFF(@matched, 1, 0)) AS match_rate + FROM @relation +) + +SELECT * +FROM final +); + +EVIDENCE ( + WITH final AS ( + SELECT * + FROM @relation + WHERE NOT COALESCE(@matched, FALSE) +) + +SELECT * +FROM final +); +""", +} + + +MACRO_EXCEPTIONS: tuple[tuple[str, str, str], ...] = ( + ( + "SQBRSQL018", + "cdc_latest", + """ +The macro cannot see its callers' keys on its own. Every call site is still checked, against the +unique, non-null _seq that each change-log source declares. +""", + ), + ( + "SQBRSQL018", + "cdc_history", + """ +The macro cannot see its callers' keys on its own. _seq is unique and non-null in every change-log +source, so ordering by it within a key and load is deterministic. +""", + ), + ( + "SQBRSQL034", + "load_metadata", + """ +A column-list macro body is a select list by construction. Its columns are inserted into the +caller's own CTE, where the rule is checked again. +""", + ), +) + + +def declare_all(project: Project) -> None: + for rule, name, reason in MACRO_EXCEPTIONS: + project.exceptions.append((rule, ("macro", name), reason)) + for name, content in SQL_MACROS.items(): + project.declare(Declaration(name, "macros/sql", content)) + for name, (file_name, content) in PY_MACROS.items(): + project.declare(Declaration(name, "macros/python", content, file_name)) + for name, content in ENUMS.items(): + project.declare(Declaration(name, "enums", content)) + for name, content in CONSTANTS.items(): + project.declare(Declaration(name, "constants", content)) + for name, content in AUDITS.items(): + project.declare(Declaration(name, "audits/generic", content)) + + +def currencies_seed() -> str: + rows = ["currency_code,minor_unit_exponent"] + rows.extend(f"{code},{exponent}" for code, exponent in sorted(CURRENCY_EXPONENT.items())) + return "\n".join(rows) + "\n" diff --git a/benchmarks/duckshop-10k/duckshop10k/domain.py b/benchmarks/duckshop-10k/duckshop10k/domain.py new file mode 100644 index 0000000000..53e1741017 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/domain.py @@ -0,0 +1,624 @@ +"""Domain-specific intermediate models that do not fit the generic templates.""" + +from __future__ import annotations + +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.staging import with_query + + +def customer_profile( + project: Project, + name: str, + folder: str, + customers: Model, + addresses: Model, + consents: Model, + store: str, +) -> Model: + ctes = [ + import_cte("customers", f'__ref("{customers.name}")'), + import_cte("addresses", f'__ref("{addresses.name}")'), + import_cte("consents", f'__ref("{consents.name}")'), + cte( + "default_addresses", + "SELECT\n a.customer_id,\n a.city,\n a.postcode,\n a.country_code\n" + "FROM addresses AS a\nWHERE\n a.is_default\n" + "QUALIFY ROW_NUMBER() OVER (\n PARTITION BY a.customer_id\n" + " ORDER BY a.valid_from_at_utc DESC, a.address_id\n) = 1", + ), + cte( + "consent_lists", + "SELECT\n c.customer_id,\n" + " ARRAY_AGG(DISTINCT c.consent_type) WITHIN GROUP (ORDER BY c.consent_type)" + " AS consent_types\nFROM consents AS c\nWHERE\n c.consent_type IS NOT NULL\n" + "GROUP BY\n c.customer_id", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "c.customer_id", + f"c.{store}", + "c.region_code", + "c.email", + "c.phone", + "c.full_name", + "c.created_at_utc", + "c.loyalty_tier", + "c.is_marketing_opt_in", + "COALESCE(d.country_code, c.country_code)::VARCHAR AS country_code", + "d.city::VARCHAR AS default_city", + "d.postcode::VARCHAR AS default_postcode", + "COALESCE(l.consent_types, ARRAY_CONSTRUCT())::ARRAY AS consent_types", + "ARRAY_CONTAINS('MARKETING'::VARIANT, COALESCE(l.consent_types," + " ARRAY_CONSTRUCT()))" + "::BOOLEAN AS has_marketing_consent", + ] + ) + + "\nFROM customers AS c\nLEFT JOIN default_addresses AS d\n" + " ON c.customer_id = d.customer_id\nLEFT JOIN consent_lists AS l\n" + " ON c.customer_id = l.customer_id", + ), + ] + columns = [ + Column("customer_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column(store, "VARCHAR"), + Column("region_code", "VARCHAR"), + Column("email", "VARCHAR"), + Column("phone", "VARCHAR"), + Column("full_name", "VARCHAR"), + Column("created_at_utc", "TIMESTAMP_NTZ"), + Column("loyalty_tier", "VARCHAR"), + Column("is_marketing_opt_in", "BOOLEAN"), + Column("country_code", "VARCHAR"), + Column("default_city", "VARCHAR"), + Column("default_postcode", "VARCHAR"), + Column("consent_types", "ARRAY"), + Column("has_marketing_consent", "BOOLEAN"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=( + "One row per customer with their current default address and the consents they " + "have granted" + ), + columns=columns, + body=with_query(ctes), + refs=[customers.name, addresses.name, consents.name], + ) + ) + + +def cart_funnel(project: Project, name: str, folder: str, carts_stg: Model, carts: Model) -> Model: + ctes = [ + import_cte("staged_carts", f'__ref("{carts_stg.name}")'), + import_cte("carts", f'__ref("{carts.name}")'), + cte( + "item_lists", + "SELECT\n s.cart_id,\n" + " FILTER(s.items, i -> COALESCE(i:quantity::INT, 0) > 0) AS live_items\n" + "FROM staged_carts AS s", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "c.cart_id", + "c.customer_id", + "c.anonymous_id", + "c.opened_at_utc", + "c.last_activity_at_utc", + "ARRAY_SIZE(l.live_items)::INT AS live_item_count", + "REDUCE(\n TRANSFORM(l.live_items, i -> COALESCE(i:quantity::INT, 1)),\n" + " 0,\n (acc, q) -> acc + q\n )::INT AS unit_count", + "ARRAY_DISTINCT(TRANSFORM(l.live_items, i -> UPPER(i:sku::VARCHAR)))::ARRAY" + " AS distinct_skus", + "DATEDIFF('minute', c.opened_at_utc, c.last_activity_at_utc)::INT" + " AS active_minutes", + "(ARRAY_SIZE(l.live_items) > 0 AND c.customer_id IS NULL)::BOOLEAN" + " AS is_anonymous_with_items", + ] + ) + + "\nFROM carts AS c\nINNER JOIN item_lists AS l\n ON c.cart_id = l.cart_id", + ), + ] + columns = [ + Column("cart_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("customer_id", "VARCHAR"), + Column("anonymous_id", "VARCHAR"), + Column("opened_at_utc", "TIMESTAMP_NTZ"), + Column("last_activity_at_utc", "TIMESTAMP_NTZ"), + Column("live_item_count", "INT"), + Column("unit_count", "INT"), + Column("distinct_skus", "ARRAY"), + Column("active_minutes", "INT"), + Column("is_anonymous_with_items", "BOOLEAN"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=( + "One row per cart with its live items, unit count and distinct SKUs, computed with " + "array lambdas over the cart payload" + ), + columns=columns, + body=with_query(ctes), + refs=[carts_stg.name, carts.name], + ) + ) + + +def stock_ledger( + project: Project, + name: str, + folder: str, + movements: Model, + adjustments: Model, + receipts: Model, + picks: Model, + warehouse_column: str, +) -> Model: + parts = [ + ("movements", movements, "movement_id", "qty_delta", "moved_at_utc", "'movement'"), + ( + "adjustments", + adjustments, + "adjustment_id", + "qty_delta", + "adjusted_at_utc", + "'adjustment'", + ), + ("receipts", receipts, "receipt_id", "quantity", "received_at_utc", "'receipt'"), + ("picks", picks, "pick_id", "-quantity", "picked_at_utc", "'pick'"), + ] + ctes = [import_cte(alias, f'__ref("{model.name}")') for alias, model, *_ in parts] + branches = [] + for alias, _model, key, quantity, moment, kind in parts: + signed = f"(-r.{quantity[1:]})::INT" if quantity.startswith("-") else f"r.{quantity}::INT" + branches.append( + "SELECT\n" + + select_list( + [ + f"({kind} || ':' || r.{key})::VARCHAR AS movement_key", + "r.sku", + f"r.{moment}::TIMESTAMP_NTZ AS moved_at_utc", + f"{signed} AS qty_delta", + f"{kind}::VARCHAR AS movement_type", + ] + ) + + f"\nFROM {alias} AS r" + ) + ctes.append(cte("ledger_rows", "\nUNION ALL\n".join(branches))) + ctes.append( + cte( + "final", + "SELECT\n" + + select_list( + [ + "l.movement_key", + "l.sku", + "l.moved_at_utc", + "l.qty_delta", + "l.movement_type", + "SUM(l.qty_delta) OVER (\n PARTITION BY l.sku\n" + " ORDER BY l.moved_at_utc, l.movement_key\n" + " ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n )::INT AS" + " on_hand_after", + "(SUM(l.qty_delta) OVER (\n PARTITION BY l.sku\n" + " ORDER BY l.moved_at_utc, l.movement_key\n" + " ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) < 0)::BOOLEAN" + " AS is_negative_stock", + ] + ) + + "\nFROM ledger_rows AS l\nWHERE\n l.sku IS NOT NULL", + ) + ) + columns = [ + Column("movement_key", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("sku", "VARCHAR", ("not_null",)), + Column("moved_at_utc", "TIMESTAMP_NTZ"), + Column("qty_delta", "INT"), + Column("movement_type", "VARCHAR"), + Column("on_hand_after", "INT"), + Column("is_negative_stock", "BOOLEAN"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=( + "Signed stock ledger of movements, adjustments, receipts and picks per" + " SKU, with the " + "running quantity on hand after each entry" + ), + columns=columns, + body=with_query(ctes), + refs=[model.name for _alias, model, *_ in parts], + ) + ) + + +def sessions(project: Project, name: str, folder: str, events: Model) -> Model: + config = [ + "constants (\n _read_margin_hours 24,\n )", + "materialized incremental", + "incremental_strategy delete_insert", + "incremental_mode microbatch", + "microbatch_strategy watermark", + "cursor_watermark_mode all", + "cursor session_day", + "cursor_type timestamp", + "cursor_grain day", + "batch_size 1d", + "lookback 3d", + "replay_on_change full", + f"cursor_inputs (\n {events.name} (column received_at, roles [watermark]),\n )", + ] + ctes = [ + import_cte("events", f'__ref("{events.name}")'), + cte( + "human_events", + "SELECT\n" + + select_list( + [ + "e.event_id", + "e.anonymous_id", + "e.customer_id", + "e.storefront_code", + "e.event_type", + "e.event_at_utc", + "e.page_path", + "e.referrer_host", + "e.client_timezone", + ] + ) + + "\nFROM events AS e\nWHERE\n" + " e.event_at_utc >= DATEADD('hour', -@const(\"_read_margin_hours\"), " + "__cursor_start())\n" + " AND e.event_at_utc < DATEADD('hour', @const(\"_read_margin_hours\"), " + "__cursor_end())\n" + " AND NOT e.is_bot\n AND e.anonymous_id IS NOT NULL", + ), + cte( + "gaps", + "SELECT\n" + + select_list( + [ + "h.event_id", + "h.anonymous_id", + "h.customer_id", + "h.storefront_code", + "h.event_type", + "h.event_at_utc", + "h.page_path", + "h.referrer_host", + "h.client_timezone", + "IFF(DATEDIFF(\n 'minute',\n LAG(h.event_at_utc) OVER (PARTITION BY " + "h.anonymous_id ORDER BY h.event_at_utc, h.event_id),\n h.event_at_utc\n" + ' ) <= @const("session_gap_minutes"), 0, 1) AS is_session_start', + ] + ) + + "\nFROM human_events AS h", + ), + cte( + "numbered", + "SELECT\n" + + select_list( + [ + "g.anonymous_id", + "g.customer_id", + "g.storefront_code", + "g.event_type", + "g.event_at_utc", + "g.page_path", + "g.referrer_host", + "g.client_timezone", + "SUM(g.is_session_start) OVER (\n PARTITION BY g.anonymous_id\n" + " ORDER BY g.event_at_utc, g.event_id\n" + " ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS session_number", + ] + ) + + "\nFROM gaps AS g", + ), + cte( + "grouped", + "SELECT\n" + + select_list( + [ + "n.anonymous_id", + "MAX(n.customer_id) AS customer_id", + "MIN_BY(n.storefront_code, n.event_at_utc) AS storefront_code", + "MIN(n.event_at_utc) AS session_start_utc", + "MAX(n.event_at_utc) AS session_end_utc", + "COUNT(*) AS event_count", + "COUNT_IF(n.event_type = 'page_view') AS page_view_count", + "MIN_BY(n.page_path, n.event_at_utc) AS landing_page_path", + "MIN_BY(n.referrer_host, n.event_at_utc) AS referrer_host", + "MIN_BY(n.client_timezone, n.event_at_utc) AS client_timezone", + "BOOLOR_AGG(n.event_type = 'add_to_cart') AS has_add_to_cart", + "BOOLOR_AGG(n.event_type = 'checkout_complete') AS has_order", + ] + ) + + "\nFROM numbered AS n\nGROUP BY\n n.anonymous_id,\n n.session_number", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "(\n s.anonymous_id || ':' || TO_CHAR(s.session_start_utc," + " 'YYYYMMDDHH24MISS')" + "\n )::VARCHAR AS session_id", + "s.anonymous_id", + "s.customer_id::VARCHAR AS customer_id", + "s.storefront_code::VARCHAR AS storefront_code", + "DATE_TRUNC('day', s.session_start_utc)::TIMESTAMP_NTZ AS session_day", + "s.session_start_utc::DATE AS session_date", + "s.session_start_utc", + "s.session_end_utc", + "CONVERT_TIMEZONE(\n 'UTC',\n COALESCE(s.client_timezone, 'UTC'),\n" + " s.session_start_utc\n )::TIMESTAMP_NTZ AS session_start_visitor_local", + "DATEDIFF('second', s.session_start_utc, s.session_end_utc)::INT" + " AS duration_seconds", + "s.event_count::INT AS event_count", + "s.page_view_count::INT AS page_view_count", + "s.landing_page_path::VARCHAR AS landing_page_path", + "s.referrer_host::VARCHAR AS referrer_host", + "(s.page_view_count <= 1 AND s.event_count <= 1)::BOOLEAN AS is_bounce", + "s.has_add_to_cart::BOOLEAN AS has_add_to_cart", + "s.has_order::BOOLEAN AS has_order", + ] + ) + + "\nFROM grouped AS s\nWHERE\n s.session_start_utc >= __cursor_start()\n" + " AND s.session_start_utc < __cursor_end()", + ), + ] + columns = [ + Column("session_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("anonymous_id", "VARCHAR", ("not_null",)), + Column("customer_id", "VARCHAR"), + Column("storefront_code", "VARCHAR"), + Column("session_day", "TIMESTAMP_NTZ", ("not_null",)), + Column("session_date", "DATE"), + Column("session_start_utc", "TIMESTAMP_NTZ"), + Column("session_end_utc", "TIMESTAMP_NTZ"), + Column("session_start_visitor_local", "TIMESTAMP_NTZ"), + Column("duration_seconds", "INT"), + Column("event_count", "INT"), + Column("page_view_count", "INT"), + Column("landing_page_path", "VARCHAR"), + Column("referrer_host", "VARCHAR"), + Column("is_bounce", "BOOLEAN"), + Column("has_add_to_cart", "BOOLEAN"), + Column("has_order", "BOOLEAN"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=( + "One row per visitor session, split wherever 30 minutes pass without activity and " + "built in daily microbatches. Each batch keeps the sessions that start inside it, " + "reading a day of events either side so sessions crossing batch edges are whole." + ), + config=config, + columns=columns, + body=with_query(ctes), + refs=[events.name], + ) + ) + + +def ageing( + project: Project, + name: str, + folder: str, + invoices: Model, + key: str, + counterparty: str, + amount: str, + entity_column: str, + description: str, +) -> Model: + buckets = ( + "CASE\n WHEN i.days_overdue <= 0 THEN 'current'\n" + " WHEN i.days_overdue <= 30 THEN '1_30'\n" + " WHEN i.days_overdue <= 60 THEN '31_60'\n" + " WHEN i.days_overdue <= 90 THEN '61_90'\n" + " ELSE 'over_90'\n END::VARCHAR AS ageing_bucket" + ) + ctes = [ + import_cte("invoices_in", f'__ref("{invoices.name}")'), + cte( + "open_invoices", + "SELECT\n" + + select_list( + [ + f"v.{key}", + f"v.{counterparty}", + f"v.{entity_column}", + "v.currency_code", + f"v.{amount}", + "v.due_date", + "DATEDIFF('day', v.due_date, '@@valuation_date'::DATE)::INT AS days_overdue", + ] + ) + + "\nFROM invoices_in AS v\nWHERE\n v.status <> 'PAID'", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + f"i.{key}", + f"i.{counterparty}", + f"i.{entity_column}", + "i.currency_code", + f"i.{amount}", + "i.due_date", + "i.days_overdue", + buckets, + f"SUM(i.{amount}) OVER (PARTITION BY i.{counterparty})::NUMBER(18, 2)" + " AS counterparty_open_amount", + ] + ) + + "\nFROM open_invoices AS i", + ), + ] + columns = [ + Column(key, "VARCHAR", ("not_null", "unique"), nullable=False), + Column(counterparty, "VARCHAR"), + Column(entity_column, "VARCHAR"), + Column("currency_code", "VARCHAR"), + Column(amount, "NUMBER(18, 2)"), + Column("due_date", "DATE"), + Column("days_overdue", "INT"), + Column( + "ageing_bucket", + "VARCHAR", + ("accepted_values (values ['current', '1_30', '31_60', '61_90', 'over_90'])",), + ), + Column("counterparty_open_amount", "NUMBER(18, 2)"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + columns=columns, + body=with_query(ctes), + refs=[invoices.name], + ) + ) + + +def customer_identity(project: Project, name: str, folder: str, customers: Model) -> Model: + ctes = [ + import_cte("customers", f'__ref("{customers.name}")'), + cte( + "keyed", + "SELECT\n" + + select_list( + [ + "c.customer_id", + "c.storefront_code", + "c.created_at_utc", + "COALESCE(c.email, c.phone, c.customer_id) AS identity_basis", + ] + ) + + "\nFROM customers AS c", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "(k.storefront_code || ':' || k.customer_id)::VARCHAR AS" + " storefront_customer_key", + "k.customer_id", + "k.storefront_code", + "HASH(k.identity_basis)::NUMBER(38, 0) AS identity_hash", + "FIRST_VALUE(k.storefront_code || ':' || k.customer_id) OVER (\n" + " PARTITION BY k.identity_basis\n ORDER BY k.created_at_utc," + " k.storefront_code, k.customer_id\n" + " )::VARCHAR AS global_customer_key", + "COUNT(*) OVER (PARTITION BY k.identity_basis)::INT AS linked_account_count", + ] + ) + + "\nFROM keyed AS k", + ), + ] + columns = [ + Column("storefront_customer_key", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("customer_id", "VARCHAR"), + Column("storefront_code", "VARCHAR"), + Column("identity_hash", "NUMBER(38, 0)"), + Column("global_customer_key", "VARCHAR", ("not_null",)), + Column("linked_account_count", "INT"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=( + "One row per storefront account linked to a global customer by email or phone. The " + "identity hash uses HASH, whose output is engine-specific, so it is an expected " + "divergence across engines." + ), + tags=["known_divergence"], + columns=columns, + body=with_query(ctes), + refs=[customers.name], + ) + ) + + +def first_response( + project: Project, + name: str, + folder: str, + tickets: Model, + comments: Model, + local_time: bool, +) -> Model: + created = "t.created_at_utc" + response = "MIN(IFF(c.author_type = 'AGENT', c.written_at_utc, NULL))" + items = [ + "t.ticket_id", + "t.priority", + "t.contact_channel", + created, + f"{response} AS first_response_at_utc", + f"DATEDIFF('minute', {created}, {response})::INT AS minutes_to_first_response", + ] + columns = [ + Column("ticket_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("priority", "INT"), + Column("contact_channel", "VARCHAR"), + Column("created_at_utc", "TIMESTAMP_NTZ"), + Column("first_response_at_utc", "TIMESTAMP_NTZ"), + Column("minutes_to_first_response", "INT"), + ] + description = "One row per ticket with the time to the first agent response" + tags: list[str] = [] + if local_time: + items.append(f"{created}::TIMESTAMP_LTZ AS created_at_session_local") + columns.append(Column("created_at_session_local", "TIMESTAMP_LTZ")) + description += ( + ". The creation time is also shown as TIMESTAMP_LTZ, which depends on the session time " + "zone, so it is an expected divergence across engines" + ) + tags = ["known_divergence"] + ctes = [ + import_cte("tickets", f'__ref("{tickets.name}")'), + import_cte("comments", f'__ref("{comments.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM tickets AS t\nLEFT JOIN comments AS c\n" + " ON t.ticket_id = c.ticket_id\nGROUP BY\n t.ticket_id,\n t.priority,\n" + " t.contact_channel,\n t.created_at_utc", + ), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + tags=tags, + columns=columns, + body=with_query(ctes), + refs=[tickets.name, comments.name], + ) + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/extras.py b/benchmarks/duckshop-10k/duckshop10k/extras.py new file mode 100644 index 0000000000..d769f8f98e --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/extras.py @@ -0,0 +1,395 @@ +"""Functions, reference sources, audit factories and the few hand-shaped models around them.""" + +from __future__ import annotations + +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.layers import Intermediate +from duckshop10k.staging import with_query +from duckshop10k.world import CURRENCY_EXPONENT + +SKU_FAMILY = """FUNCTION ( + description "Product family prefix of a canonical SKU, such as DECOY for DECOY-TEAL-08", + arguments (sku VARCHAR), + returns VARCHAR, +); + +UPPER(REGEXP_SUBSTR(sku, '^[A-Za-z]+')) +""" + +LUHN_VALID = '''from sqlbuild.functions import udf + + +@udf( + arguments={"digits": "VARCHAR"}, + returns="BOOLEAN", + runtime_version="3.11", +) +def main(digits: str | None) -> bool | None: + """Whether a string of digits passes the Luhn checksum; NULL for empty or non-numeric input.""" + if not digits or not digits.isdigit(): + return None + total = 0 + for index, char in enumerate(reversed(digits)): + value = int(char) + if index % 2 == 1: + value *= 2 + if value > 9: + value -= 9 + total += value + return total % 10 == 0 +''' + +CUSTOMER_ORDERS = """FUNCTION ( + description "Orders placed by one storefront customer with their USD value and status", + arguments (p_customer_id VARCHAR), + returns table ( + order_id VARCHAR, + storefront_code VARCHAR, + placed_at_utc TIMESTAMP_NTZ, + status VARCHAR, + gross_amount_usd "NUMBER(18, 2)" + ) +); + +WITH orders AS ( + SELECT * + FROM __ref("fct_orders") +), + +final AS ( + SELECT + o.order_id, + o.storefront_code, + o.placed_at_utc, + o.status, + o.gross_amount_usd + FROM orders AS o + WHERE o.customer_id = p_customer_id +) + +SELECT * +FROM final +""" + +FACT_QUALITY = '''"""Audit cases shared by every published fact table.""" + +from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory + + +@audit_factory +def fact_table_quality() -> list[AuditCase]: + """A fact table must never publish empty.""" + return [ + AuditCase( + name="fact_table_has_rows", + definition="has_rows", + arguments={"minimum_rows": 1}, + severity=AuditSeverity.WARN, + description="A fact table built with no rows usually means an upstream load failed.", + meta={"check": "volume"}, + ) + ] +''' + +# Reference rates to USD, invented but plausible, on three dates. +_RATES: dict[str, tuple[float, float, float]] = { + "AED": (0.27230, 0.27230, 0.27230), + "AUD": (0.61900, 0.65800, 0.66250), + "BHD": (2.65250, 2.65250, 2.65250), + "BRL": (0.16200, 0.17100, 0.16950), + "CAD": (0.69500, 0.73200, 0.73700), + "CHF": (1.10300, 1.12800, 1.13100), + "CLP": (0.00101, 0.00106, 0.00105), + "COP": (0.00023, 0.00025, 0.00025), + "CZK": (0.04120, 0.04310, 0.04290), + "DKK": (0.13870, 0.14510, 0.14620), + "EUR": (1.03500, 1.08250, 1.09100), + "GBP": (1.25100, 1.26400, 1.27150), + "HUF": (0.00252, 0.00271, 0.00274), + "INR": (0.01168, 0.01202, 0.01198), + "JPY": (0.00636, 0.00682, 0.00672), + "KRW": (0.00068, 0.00073, 0.00072), + "KWD": (3.24300, 3.25400, 3.25150), + "MXN": (0.04910, 0.05720, 0.05660), + "NOK": (0.08810, 0.09340, 0.09410), + "NZD": (0.56100, 0.60200, 0.60600), + "PLN": (0.24300, 0.25100, 0.25350), + "QAR": (0.27470, 0.27470, 0.27470), + "SAR": (0.26650, 0.26650, 0.26650), + "SEK": (0.09040, 0.09560, 0.09710), + "SGD": (0.73300, 0.74500, 0.74800), + "THB": (0.02920, 0.02810, 0.02840), + "TWD": (0.03050, 0.03120, 0.03100), + "USD": (1.00000, 1.00000, 1.00000), + "ZAR": (0.05320, 0.05510, 0.05470), +} +_RATE_DATES = ("2025-01-01", "2026-01-01", "2026-03-01") + + +def fx_source() -> str: + assert set(_RATES) == set(CURRENCY_EXPONENT) + rows = [ + f" ('{code}', '{day}', {rates[index]:.8f})" + for code, rates in sorted(_RATES.items()) + for index, day in enumerate(_RATE_DATES) + ] + return ( + "# Reference data defined inline so the project compiles without an FX feed.\n" + "sources:\n" + " - name: finance__fx_rates\n" + " description: Daily reference rates to USD per currency, defined inline as VALUES\n" + " contract: enforced\n" + " expression: |\n" + " SELECT v.currency_code, v.rate_date, v.rate_to_usd\n" + " FROM (\n" + " VALUES\n" + + ",\n".join(rows) + + "\n ) AS v (currency_code, rate_date, rate_to_usd)\n" + " columns:\n" + " - name: currency_code\n" + " type: VARCHAR\n" + " nullable: false\n" + " audits:\n" + " - not_null\n" + " - name: rate_date\n" + " type: DATE\n" + " nullable: false\n" + " audits:\n" + " - not_null\n" + " - name: rate_to_usd\n" + " type: NUMBER(18,8)\n" + " nullable: false\n" + ) + + +def build_extras(project: Project, result: Intermediate) -> None: + project.write("functions/sql/udf__sku_family.sql", SKU_FAMILY) + project.write("functions/python/luhn_valid.py", LUHN_VALID) + project.write("python/audits/fact_quality.py", FACT_QUALITY) + project.write("sources/finance_reference.yml", fx_source()) + charges = result.combined.get("payments__charges_enriched") + if charges is not None: + _card_checks(project, charges) + if "fct_orders" in project.models: + project.write("functions/sql/table_fn__customer_orders.sql", CUSTOMER_ORDERS) + _order_lookup(project) + tickets = result.combined.get("support__tickets_enriched") + if tickets is not None: + _open_ticket_age(project, tickets) + orders = result.combined.get("suppliers__purchase_orders_enriched") + if orders is not None: + _overdue_purchase_orders(project, orders) + _dim_date(project) + + +def _card_checks(project: Project, charges: Model) -> None: + items = [ + "c.charge_id", + "c.gateway_code", + "c.card_bin", + '__udf("luhn_valid")(c.card_bin)::BOOLEAN AS is_card_bin_checksum_valid', + "REGEXP_LIKE(c.card_bin, '^[0-9]{6,8}$')::BOOLEAN AS is_card_bin_well_formed", + "c.charged_amount", + ] + columns = [ + Column("charge_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("gateway_code", "VARCHAR"), + Column("card_bin", "VARCHAR"), + Column("is_card_bin_checksum_valid", "BOOLEAN"), + Column("is_card_bin_well_formed", "BOOLEAN"), + Column("charged_amount", "NUMBER(18, 2)"), + ] + project.add( + Model( + name="int_payments__card_bin_checks", + folder="models/intermediate/payments", + layer="intermediate", + description="One row per gateway charge with checks on the card number" + " prefix it reported", + columns=columns, + body=with_query( + [ + import_cte("charges", f'__ref("{charges.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM charges AS c\nWHERE\n " + " c.card_bin IS NOT NULL", + ), + ] + ), + refs=[charges.name], + ) + ) + + +def _order_lookup(project: Project) -> None: + project.add( + Model( + name="rpt_customer_order_lookup", + folder="models/marts/support", + layer="marts", + description=( + "Every order of the customer named by the lookup_customer_id variable, for " + "support agents, read through the customer orders table function" + ), + columns=[ + Column("order_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("storefront_code", "VARCHAR"), + Column("placed_at_utc", "TIMESTAMP_NTZ"), + Column("status", "VARCHAR"), + Column("gross_amount_usd", "NUMBER(18, 2)"), + ], + body=with_query( + [ + cte( + "customer_orders", + 'SELECT *\nFROM __table_fn("table_fn__customer_orders")' + "('@@lookup_customer_id')", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "o.order_id", + "o.storefront_code", + "o.placed_at_utc", + "o.status", + "o.gross_amount_usd", + ] + ) + + "\nFROM customer_orders AS o", + ), + ] + ), + ) + ) + + +def _open_ticket_age(project: Project, tickets: Model) -> None: + items = [ + "t.ticket_id", + "t.support_tool_code", + "t.contact_channel", + "t.created_at_utc", + "DATEDIFF(\n 'hour',\n t.created_at_utc,\n" + " CONVERT_TIMEZONE('UTC', CURRENT_TIMESTAMP())::TIMESTAMP_NTZ\n )::INT AS age_hours", + ] + project.add( + Model( + name="rpt_support__open_ticket_age", + folder="models/marts/support", + layer="marts", + description=( + "Tickets with no satisfaction response yet and their age in hours at query time. " + "The age reads CURRENT_TIMESTAMP(), which is session-dependent, so it is an " + "expected divergence across engines." + ), + tags=["known_divergence"], + columns=[ + Column("ticket_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("support_tool_code", "VARCHAR"), + Column("contact_channel", "VARCHAR"), + Column("created_at_utc", "TIMESTAMP_NTZ"), + Column("age_hours", "INT"), + ], + body=with_query( + [ + import_cte("tickets", f'__ref("{tickets.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM tickets AS t\nWHERE\n" + " t.csat_responses_count = 0", + ), + ] + ), + refs=[tickets.name], + ) + ) + + +def _overdue_purchase_orders(project: Project, orders: Model) -> None: + items = [ + "p.po_id", + "p.supplier_code", + "p.warehouse_code", + "p.expected_at_utc", + "DATEDIFF(\n 'day',\n p.expected_at_utc,\n" + " CONVERT_TIMEZONE('UTC', CURRENT_TIMESTAMP())::TIMESTAMP_NTZ\n )::INT AS days_overdue", + ] + project.add( + Model( + name="rpt_operations__overdue_purchase_orders", + folder="models/marts/operations", + layer="marts", + description=( + "Purchase orders past their expected arrival with no invoice yet, aged at query " + "time from CURRENT_TIMESTAMP(), which is session-dependent and an expected " + "divergence across engines" + ), + tags=["known_divergence"], + columns=[ + Column("po_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("supplier_code", "VARCHAR"), + Column("warehouse_code", "VARCHAR"), + Column("expected_at_utc", "TIMESTAMP_NTZ"), + Column("days_overdue", "INT"), + ], + body=with_query( + [ + import_cte("purchase_orders", f'__ref("{orders.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM purchase_orders AS p\nWHERE\n" + " p.invoices_count = 0\n" + " AND p.expected_at_utc < CONVERT_TIMEZONE('UTC'," + " CURRENT_TIMESTAMP())::TIMESTAMP_NTZ", + ), + ] + ), + refs=[orders.name], + ) + ) + + +def _dim_date(project: Project) -> None: + items = [ + "s.calendar_date::DATE AS calendar_date", + "YEAR(s.calendar_date)::INT AS calendar_year", + "MONTH(s.calendar_date)::INT AS calendar_month", + "DAYOFWEEKISO(s.calendar_date)::INT AS iso_day_of_week", + "@sql.week_start(s.calendar_date) AS week_start_date", + "(DAYOFWEEKISO(s.calendar_date) >= 6)::BOOLEAN AS is_weekend", + ] + project.add( + Model( + name="dim_date", + folder="models/marts/core", + layer="marts", + description="One row per calendar day of the reporting history, from GENERATOR", + config=["contract enforced"], + columns=[ + Column("calendar_date", "DATE", ("not_null", "unique"), nullable=False), + Column("calendar_year", "INT"), + Column("calendar_month", "INT"), + Column("iso_day_of_week", "INT"), + Column("week_start_date", "DATE"), + Column("is_weekend", "BOOLEAN"), + ], + body=with_query( + [ + cte( + "days", + "SELECT SEQ4() AS day_offset\nFROM TABLE(GENERATOR(ROWCOUNT => 1095))", + ), + cte( + "spine", + "SELECT\n DATEADD('day', d.day_offset, " + '@const("history_start_date")::DATE)' + "::DATE AS calendar_date\nFROM days AS d", + ), + cte("final", f"SELECT\n{select_list(items)}\nFROM spine AS s"), + ] + ), + ) + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/intermediate.py b/benchmarks/duckshop-10k/duckshop10k/intermediate.py new file mode 100644 index 0000000000..0edc5fd68f --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/intermediate.py @@ -0,0 +1,534 @@ +"""The intermediate layer: conformed per-instance models, enrichment, unions and domain logic.""" + +from __future__ import annotations + +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.specs import Field, staging_name, staging_type +from duckshop10k.staging import StagedTable, with_query +from duckshop10k.world import CURRENCY_EXPONENT, Instance + +CLASSIC_STATUS = ( + (10, "PENDING"), + (20, "PAID"), + (30, "PACKED"), + (40, "SHIPPED"), + (50, "DELIVERED"), + (80, "RETURNED"), + (90, "CANCELLED"), +) + +INSTANCE_COLUMN = { + "storefront": "storefront_code", + "marketplace": "marketplace_code", + "carrier": "carrier_code", + "warehouse": "warehouse_code", + "store": "store_code", + "supplier": "supplier_code", + "gateway": "gateway_code", + "ad_network": "ad_network_code", + "messaging": "messaging_channel_code", + "affiliate": "affiliate_network_code", + "collector": "collector_code", + "support": "support_tool_code", + "ledger": "legal_entity_code", + "wholesale": "wholesale_desk_code", + "returns": "returns_portal_code", + "catalogue": "catalogue_code", +} + + +def instance_column(instance: Instance) -> str: + return INSTANCE_COLUMN[instance.system] + + +def canonical_name(item: Field) -> str: + if item.role == "ts": + return f"{item.stem}_at_utc" + if item.role == "amount": + return f"{item.stem}_amount" + if item.role == "status": + return "status" + if item.role == "array": + return f"{item.stem}_count" + return staging_name(item, "nest") + + +def canonical_type(item: Field) -> str: + if item.role == "ts": + return "TIMESTAMP_NTZ" + if item.role == "amount": + return "NUMBER(18, 2)" + if item.role == "status": + return "VARCHAR" + if item.role == "array": + return "INT" + return staging_type(item, "nest") + + +def _utc_expression(item: Field, platform: str, timezone: str, alias: str) -> str: + column = f"{alias}.{staging_name(item, platform)}" + if staging_type(item, platform) == "TIMESTAMP_TZ": + return f"@sql.tz_to_utc({column})" + if platform == "classic" and not item.in_payload: + return f"@sql.local_to_utc({column}, {timezone})" + return f"{column}::TIMESTAMP_NTZ" + + +def _status_expression(platform: str, alias: str, orders: bool) -> str: + if platform != "classic": + return f"UPPER({alias}.status)::VARCHAR" + if orders: + whens = "\n".join( + f' WHEN {code} THEN @enum("order_status").{name}' for code, name in CLASSIC_STATUS + ) + else: + whens = "\n".join(f" WHEN {code} THEN '{name}'" for code, name in CLASSIC_STATUS) + return f"CASE {alias}.status_code\n{whens}\n ELSE 'UNKNOWN'\nEND::VARCHAR" + + +def conform(project: Project, staged: StagedTable, folder: str) -> Model: + """Instance-specific units and layouts converted to the shared canonical shape.""" + instance, archetype = staged.instance, staged.archetype + platform = instance.platform + stg = project.ref(staged.stg) + amounts = [item for item in archetype.fields if item.role == "amount"] + currency = next((item for item in archetype.fields if item.role == "currency"), None) + times = [item for item in archetype.fields if item.role == "ts"] + normalised: list[str] = [f"s.{archetype.key}"] + final: list[str] = [f"n.{archetype.key}::VARCHAR AS {archetype.key}"] + columns = [Column(archetype.key, "VARCHAR", ("not_null", "unique"), nullable=False)] + code_column = instance_column(instance) + normalised.extend(["s.instance_code", "i.region_code"]) + if times: + normalised.append("i.timezone") + final.append(f"n.instance_code AS {code_column}") + final.append("n.region_code") + columns.extend([Column(code_column, "VARCHAR"), Column("region_code", "VARCHAR")]) + needs_currency_join = bool(amounts) and currency is not None and platform != "classic" + for item in archetype.fields: + name = canonical_name(item) + if item.role == "ts": + normalised.append(f"{_utc_expression(item, platform, 'i.timezone', 's')} AS {name}") + final.append(f"n.{name}") + elif item.role == "amount": + staged_name = staging_name(item, platform) + if staged_name.endswith("_minor"): + exponent = ( + "cur.minor_unit_exponent" + if needs_currency_join + else str(CURRENCY_EXPONENT[instance.currency]) + ) + normalised.append( + f"@sql.money_from_minor(s.{staged_name}, {exponent})::NUMBER(18, 2) AS {name}" + ) + else: + normalised.append(f"s.{staged_name}::NUMBER(18, 2) AS {name}") + final.append(f"n.{name}") + elif item.role == "status": + orders = instance.system == "storefront" and archetype.table == "orders" + normalised.append(f"{_status_expression(platform, 's', orders)} AS {name}") + final.append(f"n.{name}") + elif item.role == "array": + normalised.append(f"ARRAY_SIZE(s.{staging_name(item, platform)})::INT AS {name}") + final.append(f"n.{name}") + else: + normalised.append(f"s.{staging_name(item, platform)}") + final.append(f"n.{staging_name(item, platform)}") + audits: tuple[str, ...] = () + if item.role == "amount": + audits = () + columns.append(Column(name, canonical_type(item), audits)) + if times: + first = canonical_name(times[0]) + local = f"{times[0].stem}_date_local" + final.append(f"@sql.local_business_date(n.{first}, n.timezone) AS {local}") + columns.append(Column(local, "DATE")) + normalised.append("s.source_loaded_at") + final.append("n.source_loaded_at") + columns.append(Column("source_loaded_at", "TIMESTAMP_NTZ")) + ctes = [ + import_cte("staged", f'__ref("{stg.name}")'), + import_cte("instances", '__seed("source_instances")'), + cte( + "tagged", + f"SELECT\n s.*,\n '{instance.code.upper()}'::VARCHAR AS instance_code\n" + "FROM staged AS s", + ), + ] + joins = "\nINNER JOIN instances AS i\n ON s.instance_code = i.instance_code" + if needs_currency_join: + ctes.append(import_cte("currencies", '__seed("currencies")')) + joins += ( + "\nLEFT JOIN currencies AS cur\n" + f" ON s.{staging_name(currency, platform)} = cur.currency_code" + ) + ctes.append(cte("normalised", f"SELECT\n{select_list(normalised)}\nFROM tagged AS s{joins}")) + ctes.append(cte("final", f"SELECT\n{select_list(final)}\nFROM normalised AS n")) + return project.add( + Model( + name=f"int_{instance.code}__{archetype.table}", + folder=folder, + layer="intermediate", + description=( + f"{archetype.description.capitalize()} from {instance.label} in the shared shape: " + "UTC times, amounts in major units, mapped statuses and the local business date" + ), + columns=columns, + body=with_query(ctes), + refs=[stg.name], + ) + ) + + +# --- Generic templates over conformed models -------------------------------------------------- + + +def _numeric(model: Model) -> list[Column]: + return [ + column + for column in model.columns + if column.type.startswith(("NUMBER(18, 2)", "INT")) + and not column.name.endswith(("_number", "_index", "_seq")) + ] + + +def enrich( + project: Project, + name: str, + folder: str, + header: Model, + key: str, + children: list[Model], + description: str, +) -> Model: + """A header model joined to an aggregate of each child model that carries its key.""" + ctes = [import_cte("header", f'__ref("{header.name}")')] + items = [f"h.{column.name}" for column in header.columns] + columns = [ + Column(c.name, c.type, ("not_null", "unique") if c.name == key else (), c.name != key) + for c in header.columns + ] + joins: list[str] = [] + for index, child in enumerate(children, start=1): + alias = f"c{index}" + table = child.name.split("__", 1)[1] + ctes.append(import_cte(f"{table}_rows", f'__ref("{child.name}")')) + measures = [column for column in _numeric(child) if column.name != key][:3] + aggregates = [f"r.{key}", f"COUNT(*) AS {table}_count"] + aggregates.extend(f"SUM(r.{column.name}) AS {table}_{column.name}" for column in measures) + ctes.append( + cte( + f"{table}_totals", + f"SELECT\n{select_list(aggregates)}\nFROM {table}_rows AS r\nGROUP BY\n r.{key}", + ) + ) + joins.append(f"LEFT JOIN {table}_totals AS {alias}\n ON h.{key} = {alias}.{key}") + items.append(f"COALESCE({alias}.{table}_count, 0)::INT AS {table}_count") + columns.append(Column(f"{table}_count", "INT")) + for column in measures: + out = f"{table}_{column.name}" + out_type = "NUMBER(18, 2)" if column.type.startswith("NUMBER") else "INT" + items.append(f"COALESCE({alias}.{out}, 0)::{out_type} AS {out}") + columns.append(Column(out, out_type)) + ctes.append( + cte("final", f"SELECT\n{select_list(items)}\nFROM header AS h\n" + "\n".join(joins)) + ) + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + columns=columns, + body=with_query(ctes), + refs=[header.name, *(child.name for child in children)], + ) + ) + + +def sequence( + project: Project, + name: str, + folder: str, + source: Model, + key: str, + parent: str, + moment: str, + measure: str | None, + description: str, +) -> Model: + """Per-parent ordering: sequence number, gap to the previous row and running totals.""" + items = [ + f"s.{key}", + f"s.{parent}", + f"s.{moment}", + f"ROW_NUMBER() OVER (PARTITION BY s.{parent} ORDER BY s.{moment}, s.{key})::INT" + f" AS {parent.removesuffix('_id')}_sequence_number", + f"DATEDIFF(\n 'day',\n LAG(s.{moment}) OVER (PARTITION BY s.{parent}" + f" ORDER BY s.{moment}, s.{key})," + f"\n s.{moment}\n )::INT AS days_since_previous", + f"(LAG(s.{key}) OVER (PARTITION BY s.{parent} ORDER BY s.{moment}, s.{key}) IS NULL)" + "::BOOLEAN AS is_first", + ] + columns = [ + Column(key, "VARCHAR", ("not_null", "unique"), nullable=False), + Column(parent, "VARCHAR"), + Column(moment, "TIMESTAMP_NTZ"), + Column(f"{parent.removesuffix('_id')}_sequence_number", "INT"), + Column("days_since_previous", "INT"), + Column("is_first", "BOOLEAN"), + ] + if measure is not None: + items.extend( + [ + f"s.{measure}", + f"SUM(s.{measure}) OVER (\n PARTITION BY s.{parent}\n ORDER BY" + f" s.{moment}, s.{key}" + "\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n )::NUMBER(18, 2)" + f" AS running_{measure}", + f"AVG(s.{measure}) OVER (\n PARTITION BY s.{parent}\n ORDER BY" + f" s.{moment}, s.{key}" + "\n ROWS BETWEEN 2 PRECEDING AND CURRENT ROW\n )::NUMBER(18, 2)" + f" AS trailing_three_{measure}", + ] + ) + columns.extend( + [ + Column(measure, "NUMBER(18, 2)"), + Column(f"running_{measure}", "NUMBER(18, 2)"), + Column(f"trailing_three_{measure}", "NUMBER(18, 2)"), + ] + ) + ctes = [ + import_cte("rows_in", f'__ref("{source.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM rows_in AS s\nWHERE\n s.{parent} IS NOT NULL", + ), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + columns=columns, + body=with_query(ctes), + refs=[source.name], + ) + ) + + +def daily( + project: Project, + name: str, + folder: str, + source: Model, + day: str, + dimension: str | None, + description: str, + layer: str = "intermediate", +) -> Model: + """Daily rollup of a model's counts and amounts, by an optional dimension.""" + measures = [ + column + for column in _numeric(source) + if column.type.startswith("NUMBER(18, 2)") and column.name != day + ][:4] + group = [f"s.{day}"] + ([f"s.{dimension}"] if dimension else []) + items = [*group, "COUNT(*)::INT AS row_count"] + columns = [Column(day, "DATE", ("not_null",))] + if dimension: + columns.append(Column(dimension, source.column(dimension).type)) + columns.append(Column("row_count", "INT")) + for column in measures: + items.append(f"SUM(s.{column.name})::NUMBER(18, 2) AS total_{column.name}") + items.append(f"ROUND(AVG(s.{column.name}), 2)::NUMBER(18, 2) AS average_{column.name}") + columns.extend( + [ + Column(f"total_{column.name}", "NUMBER(18, 2)"), + Column(f"average_{column.name}", "NUMBER(18, 2)"), + ] + ) + if len(measures) >= 2: + first, second = measures[0].name, measures[1].name + items.append( + f"@sql.percent_of(SUM(s.{second}), SUM(s.{first})) AS {second}_share_of_{first}" + ) + columns.append(Column(f"{second}_share_of_{first}", "NUMBER(9, 4)")) + ctes = [ + import_cte("rows_in", f'__ref("{source.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM rows_in AS s\nWHERE\n s.{day} IS NOT NULL\n" + f"GROUP BY\n" + ",\n".join(f" {item}" for item in group), + ), + ] + return project.add( + Model( + name=name, + folder=folder, + layer=layer, + description=description, + columns=columns, + body=with_query(ctes), + refs=[source.name], + ) + ) + + +def to_usd( + project: Project, + name: str, + folder: str, + source: Model, + key_columns: list[str], + day: str, + description: str, +) -> Model: + """Every amount converted to USD at the latest reference rate on or before the row's day.""" + amounts = [c for c in source.columns if c.type == "NUMBER(18, 2)"][:4] + items = [f"s.{column.name}" for column in source.columns] + items.append("fx.rate_to_usd::NUMBER(18, 8) AS rate_to_usd") + columns = [ + Column(c.name, c.type, ("not_null",) if c.name in key_columns[:1] else ()) + for c in source.columns + ] + columns.append(Column("rate_to_usd", "NUMBER(18, 8)")) + converted = [f"w.{column.name}" for column in source.columns] + ["w.rate_to_usd"] + for column in amounts: + converted.append( + f"ROUND(w.{column.name} * w.rate_to_usd, 2)::NUMBER(18, 2) AS {column.name}_usd" + ) + columns.append(Column(f"{column.name}_usd", "NUMBER(18, 2)")) + ctes = [ + import_cte("rows_in", f'__ref("{source.name}")'), + import_cte("fx_rates", '__source("finance__fx_rates")'), + cte( + "with_rates", + f"SELECT\n{select_list(items)}\nFROM rows_in AS s\n" + f"ASOF JOIN fx_rates AS fx MATCH_CONDITION (s.{day} >= fx.rate_date)\n" + " ON s.currency_code = fx.currency_code", + ), + cte("final", f"SELECT\n{select_list(converted)}\nFROM with_rates AS w"), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + config=[ + 'audits [\n match_rate (\n name "fx_rate_found",\n' + ' matched "rate_to_usd IS NOT NULL",\n minimum_samples 50,\n' + " thresholds (warn (below 99.5), error (below 95)),\n ),\n ]" + ], + columns=columns, + body=with_query(ctes), + refs=[source.name], + sources=["finance__fx_rates"], + ) + ) + + +def union( + project: Project, + name: str, + folder: str, + members: list[Model], + description: str, + key: str | None, + via_macro: bool, +) -> Model: + """UNION ALL of models that share one canonical shape.""" + shape = members[0].columns + names = [column.name for column in shape] + if via_macro: + refs = ", ".join(f'__ref("{member.name}")' for member in members) + listed = ", ".join(f'"{column}"' for column in names) + ctes = [ + cte("unioned", f"@py.union_relations(\n [{refs}],\n [{listed}],\n)"), + cte( + "final", + "SELECT\n" + + select_list(["u.source_relation", *(f"u.{column}" for column in names)]) + + "\nFROM unioned AS u", + ), + ] + columns = [Column("source_relation", "VARCHAR")] + else: + ctes = [ + import_cte(f"part_{index}", f'__ref("{m.name}")') for index, m in enumerate(members, 1) + ] + branches = [ + f"SELECT\n{select_list([f'p.{column}' for column in names])}\nFROM part_{index} AS p" + for index in range(1, len(members) + 1) + ] + ctes.append(cte("final", "\nUNION ALL\n".join(branches))) + columns = [] + for column in shape: + if column.name == key: + audits: tuple[str, ...] = ("not_null", "unique") + else: + audits = tuple(audit for audit in column.audits if audit != "unique") + columns.append(Column(column.name, column.type, audits, column.name != key)) + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + columns=columns, + body=with_query(ctes), + refs=[member.name for member in members], + ) + ) + + +def child_detail( + project: Project, + name: str, + folder: str, + child: Model, + parent: Model, + key: str, + description: str, +) -> Model: + """Array elements joined to their conformed parent, with per-parent element ranking.""" + child_columns = [c for c in child.columns if c.name not in (key, "element_index")] + parent_columns = [ + c + for c in parent.columns + if c.name != key + and c.type in ("TIMESTAMP_NTZ", "DATE", "VARCHAR") + and not c.name.endswith("_code") + ][:3] + items = [f"e.{c.name}" for c in child_columns] + [f"e.{key}", "e.element_index"] + items.extend(f"p.{c.name} AS parent_{c.name}" for c in parent_columns) + items.append(f"COUNT(*) OVER (PARTITION BY e.{key})::INT AS elements_on_parent") + columns = [Column(c.name, c.type, c.audits, c.nullable) for c in child_columns] + [ + Column(key, "VARCHAR"), + Column("element_index", "INT"), + ] + columns.extend(Column(f"parent_{c.name}", c.type) for c in parent_columns) + columns.append(Column("elements_on_parent", "INT")) + ctes = [ + import_cte("elements", f'__ref("{child.name}")'), + import_cte("parents", f'__ref("{parent.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM elements AS e\nINNER JOIN parents AS p\n" + f" ON e.{key} = p.{key}", + ), + ] + return project.add( + Model( + name=name, + folder=folder, + layer="intermediate", + description=description, + columns=columns, + body=with_query(ctes), + refs=[child.name, parent.name], + ) + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/layers.py b/benchmarks/duckshop-10k/duckshop10k/layers.py new file mode 100644 index 0000000000..4513d958cd --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/layers.py @@ -0,0 +1,550 @@ +"""Which intermediate models each source system gets, and the unions that combine instances.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from duckshop10k import domain +from duckshop10k.core import Model, Project +from duckshop10k.intermediate import ( + child_detail, + conform, + daily, + enrich, + instance_column, + sequence, + to_usd, + union, +) +from duckshop10k.staging import StagedTable +from duckshop10k.world import PRODUCT_LINES, REGIONS, Instance + + +@dataclass(frozen=True) +class Plan: + conform: tuple[str, ...] + enrich: tuple[tuple[str, str, tuple[str, ...]], ...] = () + sequences: tuple[tuple[str, str, str, str, str | None], ...] = () + daily: tuple[tuple[str, str, str | None], ...] = () + children: tuple[tuple[str, str, str], ...] = () + unions: tuple[str, ...] = () + + +PLANS: dict[str, Plan] = { + "storefront": Plan( + conform=( + "customers", + "customer_addresses", + "orders", + "order_lines", + "payments", + "refunds", + "carts", + "promotion_redemptions", + ), + enrich=(("orders", "order_id", ("order_lines", "payments", "promotion_redemptions")),), + sequences=(("orders", "order_id", "customer_id", "placed_at_utc", "gross_amount"),), + daily=( + ("orders_enriched", "placed_date_local", "channel"), + ("payments", "authorised_date_local", "method"), + ), + children=(("order_lines", "discounts", "order_line_id"),), + unions=( + "orders_enriched", + "orders_by_customer", + "orders_enriched_daily", + "payments_daily", + "customer_profile", + "order_lines", + "cart_funnel", + "refunds", + ), + ), + "marketplace": Plan( + conform=("orders", "order_items", "fees", "returns", "shipments", "listings"), + enrich=( + ( + "orders", + "partner_order_id", + ("order_items", "fees", "returns", "shipments"), + ), + ), + daily=(("orders_enriched", "placed_date_local", "status"),), + unions=("orders_enriched", "orders_enriched_daily", "listings"), + ), + "carrier": Plan( + conform=("shipments", "tracking_events", "invoices"), + enrich=(("shipments", "shipment_id", ("tracking_events", "invoices")),), + daily=(("shipments_enriched", "shipped_date_local", "service_level"),), + children=(("shipments", "tracking", "shipment_id"),), + unions=("shipments_enriched", "shipments_enriched_daily", "shipments_tracking"), + ), + "warehouse": Plan( + conform=("receipts", "picks", "adjustments", "stock_movements", "putaways"), + enrich=(("receipts", "receipt_id", ("putaways",)),), + daily=(("picks", "picked_date_local", None),), + unions=("stock_ledger", "receipts_enriched", "picks_daily"), + ), + "store": Plan( + conform=("transactions", "transaction_lines", "tenders"), + enrich=(("transactions", "txn_id", ("transaction_lines", "tenders")),), + daily=(("transactions_enriched", "opened_date_local", "register_id"),), + unions=("transactions_enriched", "transactions_enriched_daily"), + ), + "supplier": Plan( + conform=("purchase_orders", "invoices"), + enrich=(("purchase_orders", "po_id", ("invoices",)),), + children=(("purchase_orders", "lines", "po_id"),), + unions=("purchase_orders_enriched", "purchase_orders_lines"), + ), + "gateway": Plan( + conform=("charges", "refunds", "disputes", "fees"), + enrich=(("charges", "charge_id", ("refunds", "disputes", "fees")),), + daily=(("charges_enriched", "created_date_local", "status"),), + unions=("charges_enriched", "charges_enriched_daily"), + ), + "ad_network": Plan( + conform=("campaigns", "ad_groups", "daily_spend", "conversions"), + enrich=(("campaigns", "campaign_id", ("ad_groups", "daily_spend", "conversions")),), + unions=("campaigns_enriched", "daily_spend_usd"), + ), + "messaging": Plan( + conform=("sends", "opens", "clicks", "bounces"), + enrich=(("sends", "send_id", ("opens", "clicks", "bounces")),), + unions=("sends_enriched",), + ), + "affiliate": Plan( + conform=("clicks", "conversions"), + enrich=(("clicks", "affiliate_click_id", ("conversions",)),), + unions=("clicks_enriched",), + ), + "support": Plan( + conform=("tickets", "ticket_comments", "csat_responses", "ticket_events"), + enrich=(("tickets", "ticket_id", ("ticket_comments", "csat_responses", "ticket_events")),), + unions=("tickets_enriched", "first_response"), + ), + "ledger": Plan( + conform=("gl_entries", "ap_invoices", "ar_invoices", "budgets"), + daily=(("gl_entries", "posted_date_local", "account_code"),), + unions=("gl_entries", "gl_entries_daily", "ap_ageing", "ar_ageing", "budgets"), + ), + "wholesale": Plan( + conform=("accounts", "orders", "order_lines", "invoices", "payments"), + enrich=( + ("orders", "b2b_order_id", ("order_lines", "invoices")), + ("invoices", "b2b_invoice_id", ("payments",)), + ), + unions=("orders_enriched", "invoices_enriched"), + ), + "returns": Plan( + conform=("return_requests", "return_labels", "inspections", "refunds_issued"), + enrich=( + ( + "return_requests", + "return_request_id", + ("return_labels", "inspections", "refunds_issued"), + ), + ), + unions=("return_requests_enriched",), + ), + "catalogue": Plan( + conform=("products", "product_variants", "categories", "price_lists", "media"), + enrich=(("products", "product_id", ("product_variants", "media")),), + ), +} + +AREA = { + "storefront": "storefronts", + "marketplace": "marketplaces", + "carrier": "carriers", + "warehouse": "warehouses", + "store": "stores", + "supplier": "suppliers", + "gateway": "payments", + "ad_network": "advertising", + "messaging": "messaging", + "affiliate": "affiliates", + "collector": "clickstream", + "support": "support", + "ledger": "ledgers", + "wholesale": "wholesale", + "returns": "returns", + "catalogue": "catalogue", +} + +# Unions that go through the Python union macro rather than hand-written UNION ALL branches. +MACRO_UNIONS = frozenset({"orders_enriched", "shipments_enriched", "charges_enriched"}) + + +@dataclass +class Intermediate: + """Models the mart layer reads, by instance code and by area.""" + + per_instance: dict[str, dict[str, Model]] = field(default_factory=dict) + regional: dict[str, dict[str, Model]] = field(default_factory=dict) + combined: dict[str, Model] = field(default_factory=dict) + + +def _instance_models( + project: Project, instance: Instance, tables: dict[str, StagedTable], folder: str +) -> dict[str, Model]: + plan = PLANS[instance.system] + models: dict[str, Model] = {} + prefix = f"int_{instance.code}__" + for table in plan.conform: + models[table] = conform(project, tables[table], folder) + for header, key, children in plan.enrich: + models[f"{header}_enriched"] = enrich( + project, + f"{prefix}{header}_enriched", + folder, + models[header], + key, + [models[child] for child in children], + f"One row per {header.removesuffix('s').replace('_', ' ')} from {instance.label} " + f"with totals from its {', '.join(c.replace('_', ' ') for c in children)}", + ) + for table, key, parent, moment, measure in plan.sequences: + models[f"{table}_by_{parent.removesuffix('_id')}"] = sequence( + project, + f"{prefix}{table}_by_{parent.removesuffix('_id')}", + folder, + models[table], + key, + parent, + moment, + measure, + f"{table.replace('_', ' ').capitalize()} from {instance.label} in order per " + f"{parent.removesuffix('_id')}, with gaps between them and running totals", + ) + for source, day, dimension in plan.daily: + models[f"{source}_daily"] = daily( + project, + f"{prefix}{source}_daily", + folder, + models[source], + day, + dimension, + f"Daily totals of {source.replace('_', ' ')} from {instance.label}" + + (f" by {dimension.replace('_', ' ')}" if dimension else ""), + ) + for parent, child, key in plan.children: + staged_child = project.ref(tables[parent].children[child]) + models[f"{parent}_{child}"] = child_detail( + project, + f"{prefix}{parent}_{child}", + folder, + staged_child, + models[parent], + key, + f"{child.capitalize()} on {parent.replace('_', ' ')} from {instance.label}, with the " + "parent's times and the number of elements per parent", + ) + if instance.system == "storefront": + models["customer_profile"] = domain.customer_profile( + project, + f"{prefix}customer_profile", + folder, + models["customers"], + models["customer_addresses"], + project.ref(tables["customers"].children["consents"]), + instance_column(instance), + ) + models["cart_funnel"] = domain.cart_funnel( + project, + f"{prefix}cart_funnel", + folder, + project.ref(tables["carts"].stg), + models["carts"], + ) + if instance.system == "warehouse": + models["stock_ledger"] = domain.stock_ledger( + project, + f"{prefix}stock_ledger", + folder, + models["stock_movements"], + models["adjustments"], + models["receipts"], + models["picks"], + instance_column(instance), + ) + if instance.system == "ad_network": + models["daily_spend_usd"] = to_usd( + project, + f"{prefix}daily_spend_usd", + folder, + models["daily_spend"], + ["spend_id"], + "spend_date", + f"Daily spend from {instance.label} converted to USD at the reference rate of the day", + ) + if instance.system == "support": + models["first_response"] = domain.first_response( + project, + f"{prefix}first_response", + folder, + models["tickets"], + models["ticket_comments"], + local_time=instance.code == "support_voice", + ) + if instance.system == "ledger": + models["ap_ageing"] = domain.ageing( + project, + f"{prefix}ap_ageing", + folder, + models["ap_invoices"], + "ap_invoice_id", + "supplier_ref", + "invoiced_total_amount", + instance_column(instance), + f"Open payable invoices of {instance.label} by days overdue at the valuation date", + ) + models["ar_ageing"] = domain.ageing( + project, + f"{prefix}ar_ageing", + folder, + models["ar_invoices"], + "ar_invoice_id", + "customer_ref", + "invoiced_total_amount", + instance_column(instance), + f"Open receivable invoices of {instance.label} by days overdue at the valuation date", + ) + return models + + +def _key_of(model: Model) -> str | None: + return next((c.name for c in model.columns if "unique" in c.audits), None) + + +def build_intermediate( + project: Project, + tables: dict[str, dict[str, StagedTable]], + events: dict[str, tuple[str, list[str]]], + systems: dict[str, tuple[Instance, ...]], +) -> Intermediate: + result = Intermediate() + for system, values in systems.items(): + if system == "collector" or not values: + continue + area = AREA[system] + for instance in values: + folder = f"models/intermediate/{area}/{instance.code}" + result.per_instance[instance.code] = _instance_models( + project, instance, tables[instance.code], folder + ) + plan = PLANS[system] + for entity in plan.unions: + members_by_region: dict[str, list[Model]] = {} + for instance in values: + members_by_region.setdefault(instance.region, []).append( + result.per_instance[instance.code][entity] + ) + via_macro = entity in MACRO_UNIONS + if system == "storefront": + regional: list[Model] = [] + for region in REGIONS: + members = members_by_region.get(region.code, []) + if not members: + continue + model = union( + project, + f"int_{area}_{region.code}__{entity}", + f"models/intermediate/{area}/regions", + members, + f"{entity.replace('_', ' ').capitalize()} from every {region.name} " + "storefront", + _key_of(members[0]), + via_macro, + ) + result.regional.setdefault(region.code, {})[entity] = model + regional.append(model) + members = regional + via_macro = False + else: + members = [result.per_instance[instance.code][entity] for instance in values] + if len(members) == 1: + result.combined[f"{area}__{entity}"] = members[0] + continue + result.combined[f"{area}__{entity}"] = union( + project, + f"int_{area}__{entity}", + f"models/intermediate/{area}", + members, + f"{entity.replace('_', ' ').capitalize()} from every {area.removesuffix('s')} " + "instance", + _key_of(members[0]), + via_macro, + ) + _storefront_globals(project, result) + _clickstream(project, result, events, systems.get("collector", ())) + return result + + +def _storefront_globals(project: Project, result: Intermediate) -> None: + folder = "models/intermediate/storefronts" + orders = result.combined["storefronts__orders_enriched"] + result.combined["orders_usd"] = to_usd( + project, + "int_orders_usd", + folder, + orders, + ["order_id"], + "placed_date_local", + "Every storefront order with its amounts converted to USD at the reference rate on or " + "before the local order date", + ) + result.combined["customer_identity"] = domain.customer_identity( + project, + "int_customer_identity", + "models/intermediate/customers", + result.combined["storefronts__customer_profile"], + ) + catalogue = result.per_instance.get("pim") + if catalogue is None: + return + lines = result.combined["storefronts__order_lines"] + result.combined["order_lines_catalogued"] = _catalogued_lines( + project, folder, lines, catalogue["product_variants"], catalogue["products"] + ) + for line in PRODUCT_LINES: + result.combined[f"{line}__daily_demand"] = _line_demand( + project, line, result.combined["order_lines_catalogued"] + ) + + +def _catalogued_lines( + project: Project, folder: str, lines: Model, variants: Model, products: Model +) -> Model: + from duckshop10k.core import Column, cte, import_cte, select_list + from duckshop10k.staging import with_query + + items = [ + "l.order_line_id", + "l.order_id", + "l.storefront_code", + "l.sku", + "l.quantity", + "l.unit_price_amount", + "(l.quantity * l.unit_price_amount)::NUMBER(18, 2) AS line_amount", + "v.product_id", + "v.colour", + "p.product_line", + "@udf_call AS sku_family", + "SPLIT_PART(p.category_path, '/', 1)::VARCHAR AS top_category", + ] + items[10] = '__udf("udf__sku_family")(l.sku)::VARCHAR AS sku_family' + ctes = [ + import_cte("order_lines", f'__ref("{lines.name}")'), + import_cte("variants", f'__ref("{variants.name}")'), + import_cte("products", f'__ref("{products.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM order_lines AS l\nLEFT JOIN variants AS v\n" + " ON l.sku = v.sku\nLEFT JOIN products AS p\n ON v.product_id = p.product_id", + ), + ] + columns = [ + Column("order_line_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("order_id", "VARCHAR"), + Column("storefront_code", "VARCHAR"), + Column("sku", "VARCHAR"), + Column("quantity", "INT", ("non_negative",)), + Column("unit_price_amount", "NUMBER(18, 2)"), + Column("line_amount", "NUMBER(18, 2)"), + Column("product_id", "VARCHAR"), + Column("colour", "VARCHAR"), + Column("product_line", "VARCHAR"), + Column("sku_family", "VARCHAR"), + Column("top_category", "VARCHAR"), + ] + return project.add( + Model( + name="int_order_lines_catalogued", + folder=folder, + layer="intermediate", + description=( + "Every storefront order line with its product, colour, product line," + " SKU family and " + "top-level category" + ), + columns=columns, + body=with_query(ctes), + refs=[lines.name, variants.name, products.name], + ) + ) + + +def _line_demand(project: Project, line: str, lines: Model) -> Model: + from duckshop10k.core import Column, cte, import_cte, select_list + from duckshop10k.staging import with_query + + items = [ + "l.storefront_code", + "l.sku", + "COUNT(DISTINCT l.order_id)::INT AS order_count", + "SUM(l.quantity)::INT AS units", + "SUM(l.line_amount)::NUMBER(18, 2) AS line_amount", + "RANK() OVER (\n PARTITION BY l.storefront_code\n ORDER BY" + " SUM(l.quantity) DESC, l.sku\n" + " )::INT AS units_rank_in_storefront", + ] + ctes = [ + import_cte("order_lines", f'__ref("{lines.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM order_lines AS l\nWHERE\n" + f" l.product_line = '{line.upper()}'\nGROUP BY\n l.storefront_code,\n l.sku", + ), + ] + columns = [ + Column("storefront_code", "VARCHAR", ("not_null",)), + Column("sku", "VARCHAR", ("not_null",)), + Column("order_count", "INT"), + Column("units", "INT", ("non_negative",)), + Column("line_amount", "NUMBER(18, 2)"), + Column("units_rank_in_storefront", "INT"), + ] + return project.add( + Model( + name=f"int_{line}__sku_demand", + folder=f"models/intermediate/product_lines/{line}", + layer="intermediate", + description=( + f"Units and sales of every {line.title()} product-line SKU per storefront, ranked " + "within the storefront" + ), + columns=columns, + body=with_query(ctes), + refs=[lines.name], + ) + ) + + +def _clickstream( + project: Project, + result: Intermediate, + events: dict[str, tuple[str, list[str]]], + collectors: tuple[Instance, ...], +) -> None: + session_models: list[Model] = [] + for collector in collectors: + stream, _typed = events[collector.code] + model = domain.sessions( + project, + f"int_{collector.code}__sessions", + f"models/intermediate/clickstream/{collector.code}", + project.ref(stream), + ) + result.per_instance.setdefault(collector.code, {})["sessions"] = model + session_models.append(model) + if len(session_models) > 1: + result.combined["clickstream__sessions"] = union( + project, + "int_clickstream__sessions", + "models/intermediate/clickstream", + session_models, + "Sessions from every web and app collector", + "session_id", + False, + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/manifest.py b/benchmarks/duckshop-10k/duckshop10k/manifest.py new file mode 100644 index 0000000000..f2b3bf8809 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/manifest.py @@ -0,0 +1,51 @@ +"""Write a generated project and fingerprint it with a manifest of file hashes.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from duckshop10k import GENERATOR_VERSION + +MANIFEST_NAME = "MANIFEST.sha256" + + +def manifest(files: dict[str, str]) -> str: + """One ` ` line per generated file, sorted by path.""" + lines = [ + f"{hashlib.sha256(content.encode('utf-8')).hexdigest()} {path}" + for path, content in sorted(files.items()) + ] + return "\n".join(lines) + "\n" + + +def checksum(files: dict[str, str]) -> str: + """The checksum recorded in checksum.txt: the SHA-256 of the manifest.""" + return hashlib.sha256(manifest(files).encode("utf-8")).hexdigest() + + +def checksum_line(files: dict[str, str]) -> str: + return f"generator {GENERATOR_VERSION} files {len(files)} sha256 {checksum(files)}\n" + + +def write_project(files: dict[str, str], out: Path) -> None: + """Write every file below `out`, which must not exist or must be empty.""" + if out.exists() and any(out.iterdir()): + raise SystemExit(f"output directory {out} is not empty") + for path, content in files.items(): + target = out / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content.encode("utf-8")) + (out / MANIFEST_NAME).write_bytes(manifest(files).encode("utf-8")) + + +def read_project(out: Path) -> dict[str, str]: + """Every generated file below `out`, skipping the manifest and local build output.""" + skipped = {MANIFEST_NAME} + local = ("target/", "logs/", ".sqlbuild/", "sqlbuild_local.toml") + files: dict[str, str] = {} + for path in sorted(out.rglob("*")): + relative = path.relative_to(out).as_posix() + if path.is_file() and relative not in skipped and not relative.startswith(local): + files[relative] = path.read_bytes().decode("utf-8") + return files diff --git a/benchmarks/duckshop-10k/duckshop10k/mart_layer.py b/benchmarks/duckshop-10k/duckshop10k/mart_layer.py new file mode 100644 index 0000000000..5c9272f273 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/mart_layer.py @@ -0,0 +1,532 @@ +"""Which marts each instance, team and region gets, plus published facts, dimensions, snapshots.""" + +from __future__ import annotations + +from collections.abc import Callable + +from duckshop10k import chains, marts +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.layers import AREA, Intermediate +from duckshop10k.staging import with_query +from duckshop10k.world import PRODUCT_LINES, REGIONS, Instance + +# (metric, template, source entity, arguments) +Spec = tuple[str, str, str, tuple[object, ...]] + +INSTANCE_MARTS: dict[str, tuple[Spec, ...]] = { + "storefront": ( + ("sales_weekly", "period", "orders_enriched_daily", ("week",)), + ("sales_monthly", "period", "orders_enriched_daily", ("month",)), + ("channel_mix", "share", "orders_enriched_daily", ("channel",)), + ("payment_method_mix", "share", "payments_daily", ("method",)), + ("payments_weekly", "period", "payments_daily", ("week",)), + ("payments_monthly", "period", "payments_daily", ("month",)), + ("top_customers", "top", "orders_by_customer", ("customer_id", 100)), + ("customer_cohorts", "cohorts", "orders_by_customer", ("customer_id", "placed_at_utc")), + ("discounts_by_code", "share", "order_lines_discounts", ("promo_code",)), + ("order_status_mix", "breakdown", "orders_enriched", ("status",)), + ("refund_reasons", "classify", "refunds", ("reason_text",)), + ("utm_source_mix", "share", "orders_enriched", ("utm_source",)), + ("utm_campaign_top", "top", "orders_enriched", ("utm_campaign", 25)), + ("device_mix", "share", "orders_enriched", ("device_type",)), + ("currency_mix", "share", "orders_enriched", ("currency_code",)), + ("cart_top_visitors", "top", "cart_funnel", ("anonymous_id", 50)), + ("refunds_weekly", "period", "refunds", ("week",)), + ("refund_initiators", "share", "refunds", ("initiated_by",)), + ("orders_by_channel_monthly", "breakdown", "orders_enriched", ("channel",)), + ("gift_order_share", "share", "orders_enriched", ("is_gift",)), + ), + "marketplace": ( + ("orders_weekly", "period", "orders_enriched_daily", ("week",)), + ("orders_monthly", "period", "orders_enriched_daily", ("month",)), + ("status_mix", "share", "orders_enriched_daily", ("status",)), + ("top_buyers", "top", "orders_enriched", ("buyer_ref", 100)), + ("listing_status_mix", "share", "listings", ("listing_status",)), + ("ship_country_mix", "share", "orders_enriched", ("ship_country",)), + ), + "carrier": ( + ("shipments_weekly", "period", "shipments_enriched_daily", ("week",)), + ("shipments_monthly", "period", "shipments_enriched_daily", ("month",)), + ("service_level_mix", "share", "shipments_enriched_daily", ("service_level",)), + ("top_warehouses", "top", "shipments_enriched", ("warehouse_code", 20)), + ("tracking_event_mix", "breakdown", "shipments_tracking", ("event_code",)), + ), + "warehouse": ( + ("picks_weekly", "period", "picks_daily", ("week",)), + ("picks_monthly", "period", "picks_daily", ("month",)), + ("movement_type_mix", "share", "stock_ledger", ("movement_type",)), + ("top_moving_skus", "top", "stock_ledger", ("sku", 100)), + ("receipts_monthly", "period", "receipts_enriched", ("month",)), + ("adjustment_reasons", "share", "adjustments", ("reason_code",)), + ("movement_reasons", "breakdown", "stock_movements", ("reason_code",)), + ), + "store": ( + ("sales_weekly", "period", "transactions_enriched_daily", ("week",)), + ("sales_monthly", "period", "transactions_enriched_daily", ("month",)), + ("register_mix", "share", "transactions_enriched_daily", ("register_id",)), + ("status_mix", "breakdown", "transactions_enriched", ("status",)), + ("tender_mix", "share", "tenders", ("tender_type",)), + ("top_skus", "top", "transaction_lines", ("sku", 50)), + ), + "supplier": ( + ("purchase_orders_monthly", "period", "purchase_orders_enriched", ("month",)), + ("top_skus", "top", "purchase_orders_lines", ("sku", 50)), + ("warehouse_mix", "share", "purchase_orders_enriched", ("warehouse_code",)), + ), + "gateway": ( + ("charges_weekly", "period", "charges_enriched_daily", ("week",)), + ("status_mix", "share", "charges_enriched_daily", ("status",)), + ("top_orders", "top", "charges_enriched", ("order_id", 100)), + ), + "ad_network": ( + ("spend_weekly", "period", "daily_spend_usd", ("week",)), + ("spend_monthly", "period", "daily_spend_usd", ("month",)), + ("objective_mix", "share", "campaigns_enriched", ("objective",)), + ("top_campaigns", "top", "campaigns_enriched", ("campaign_id", 25)), + ), + "messaging": ( + ("sends_weekly", "period", "sends_enriched", ("week",)), + ("top_campaigns", "top", "sends_enriched", ("campaign_id", 50)), + ), + "affiliate": ( + ("clicks_weekly", "period", "clicks_enriched", ("week",)), + ("top_affiliates", "top", "clicks_enriched", ("affiliate_id", 50)), + ), + "support": ( + ("tickets_weekly", "period", "tickets_enriched", ("week",)), + ("channel_mix", "share", "tickets_enriched", ("contact_channel",)), + ("priority_mix", "breakdown", "tickets_enriched", ("priority",)), + ("response_by_channel", "share", "first_response", ("contact_channel",)), + ), + "ledger": ( + ("gl_monthly", "period", "gl_entries_daily", ("month",)), + ("account_mix", "share", "gl_entries_daily", ("account_code",)), + ("payables_by_bucket", "share", "ap_ageing", ("ageing_bucket",)), + ("receivables_by_bucket", "share", "ar_ageing", ("ageing_bucket",)), + ("budget_by_cost_centre", "share", "budgets", ("cost_centre",)), + ), + "wholesale": ( + ("orders_monthly", "period", "orders_enriched", ("month",)), + ("top_accounts", "top", "orders_enriched", ("account_id", 50)), + ("status_mix", "breakdown", "orders_enriched", ("status",)), + ("invoices_weekly", "period", "invoices_enriched", ("week",)), + ), + "returns": ( + ("requests_weekly", "period", "return_requests_enriched", ("week",)), + ("reason_classes", "classify", "return_requests", ("reason_text",)), + ), + "collector": ( + ("sessions_weekly", "period", "sessions", ("week",)), + ("landing_page_mix", "share", "sessions", ("landing_page_path",)), + ("referrer_mix", "share", "sessions", ("referrer_host",)), + ("top_customers", "top", "sessions", ("customer_id", 100)), + ), +} + +TEAM_REGION_MARTS: dict[str, tuple[Spec, ...]] = { + "finance": ( + ("revenue_weekly", "period", "orders_enriched_daily", ("week",)), + ("revenue_monthly", "period", "orders_enriched_daily", ("month",)), + ("payments_monthly", "period", "payments_daily", ("month",)), + ("payments_weekly", "period", "payments_daily", ("week",)), + ("payment_method_share", "share", "payments_daily", ("method",)), + ("refund_reasons", "classify", "refunds", ("reason_text",)), + ("storefront_share", "share", "orders_enriched", ("storefront_code",)), + ("order_status_monthly", "breakdown", "orders_enriched", ("status",)), + ("currency_share", "share", "orders_enriched", ("currency_code",)), + ), + "marketing": ( + ("channel_share", "share", "orders_enriched_daily", ("channel",)), + ("utm_source_share", "share", "orders_enriched", ("utm_source",)), + ("utm_campaign_top", "top", "orders_enriched", ("utm_campaign", 50)), + ("device_share", "share", "orders_enriched", ("device_type",)), + ("customer_cohorts", "cohorts", "orders_by_customer", ("customer_id", "placed_at_utc")), + ("top_customers", "top", "orders_by_customer", ("customer_id", 200)), + ("cart_top_customers", "top", "cart_funnel", ("customer_id", 100)), + ("orders_weekly", "period", "orders_enriched_daily", ("week",)), + ("refunds_weekly", "period", "refunds", ("week",)), + ), + "merchandising": ( + ("top_skus", "top", "order_lines", ("sku", 100)), + ("storefront_line_share", "share", "order_lines", ("storefront_code",)), + ("top_orders_by_lines", "top", "order_lines", ("order_id", 100)), + ("cart_skus_by_visitor", "top", "cart_funnel", ("anonymous_id", 100)), + ("refund_share_by_storefront", "share", "refunds", ("storefront_code",)), + ("orders_by_channel_monthly", "breakdown", "orders_enriched", ("channel",)), + ), + "support": ( + ("refund_reasons", "classify", "refunds", ("reason_text",)), + ("refunds_by_initiator", "share", "refunds", ("initiated_by",)), + ("refunds_monthly", "breakdown", "refunds", ("initiated_by",)), + ), +} + +COMBINED_MARTS: dict[str, tuple[Spec, ...]] = { + "operations": ( + ("shipments_weekly", "period", "carriers__shipments_enriched_daily", ("week",)), + ("service_level_share", "share", "carriers__shipments_enriched_daily", ("service_level",)), + ("carrier_share", "share", "carriers__shipments_enriched", ("carrier_code",)), + ("tracking_events_monthly", "breakdown", "carriers__shipments_tracking", ("event_code",)), + ("picks_weekly", "period", "warehouses__picks_daily", ("week",)), + ("movement_type_share", "share", "warehouses__stock_ledger", ("movement_type",)), + ("top_moving_skus", "top", "warehouses__stock_ledger", ("sku", 200)), + ("receipts_monthly", "period", "warehouses__receipts_enriched", ("month",)), + ("store_sales_weekly", "period", "stores__transactions_enriched_daily", ("week",)), + ("purchase_orders_monthly", "period", "suppliers__purchase_orders_enriched", ("month",)), + ("supplier_top_skus", "top", "suppliers__purchase_orders_lines", ("sku", 200)), + ("returns_weekly", "period", "returns__return_requests_enriched", ("week",)), + ), + "support": ( + ("tickets_weekly", "period", "support__tickets_enriched", ("week",)), + ("channel_share", "share", "support__tickets_enriched", ("contact_channel",)), + ("response_by_channel", "share", "support__first_response", ("contact_channel",)), + ), + "marketing": ( + ("ad_spend_weekly", "period", "advertising__daily_spend_usd", ("week",)), + ("ad_spend_monthly", "period", "advertising__daily_spend_usd", ("month",)), + ("ad_network_share", "share", "advertising__campaigns_enriched", ("ad_network_code",)), + ("messaging_sends_weekly", "period", "messaging__sends_enriched", ("week",)), + ("affiliate_clicks_weekly", "period", "affiliates__clicks_enriched", ("week",)), + ("sessions_weekly", "period", "clickstream__sessions", ("week",)), + ("landing_page_share", "share", "clickstream__sessions", ("landing_page_path",)), + ), + "finance": ( + ("gl_monthly", "period", "ledgers__gl_entries_daily", ("month",)), + ("payables_by_bucket", "share", "ledgers__ap_ageing", ("ageing_bucket",)), + ("receivables_by_bucket", "share", "ledgers__ar_ageing", ("ageing_bucket",)), + ("gateway_charges_weekly", "period", "payments__charges_enriched_daily", ("week",)), + ("marketplace_orders_monthly", "period", "marketplaces__orders_enriched_daily", ("month",)), + ("wholesale_orders_monthly", "period", "wholesale__orders_enriched", ("month",)), + ("orders_usd_monthly", "period", "orders_usd", ("month",)), + ), + "merchandising": ( + ("product_line_share", "share", "order_lines_catalogued", ("product_line",)), + ("sku_family_share", "share", "order_lines_catalogued", ("sku_family",)), + ("top_category_share", "share", "order_lines_catalogued", ("top_category",)), + ("colour_share", "share", "order_lines_catalogued", ("colour",)), + ("listing_status_share", "share", "marketplaces__listings", ("listing_status",)), + ), +} + + +def _apply( + project: Project, name: str, folder: str, spec: Spec, source: Model, subject: str +) -> Model | None: + _metric, template, _entity, args = spec + templates: dict[str, Callable[..., Model | None]] = { + "period": marts.period_rollup, + "share": marts.share_by, + "top": marts.top_n, + "breakdown": marts.breakdown, + "cohorts": marts.cohorts, + "classify": marts.classify_text, + } + return templates[template](project, name, folder, source, *args, subject) + + +def build_marts( + project: Project, result: Intermediate, systems: dict[str, tuple[Instance, ...]] +) -> dict[str, Model]: + published: dict[str, Model] = {} + for system, values in systems.items(): + for instance in values: + models = result.per_instance.get(instance.code, {}) + folder = f"models/marts/{AREA[system]}/{instance.code}" + for spec in INSTANCE_MARTS.get(system, ()): + source = models.get(spec[2]) + if source is not None: + _apply( + project, + f"rpt_{instance.code}__{spec[0]}", + folder, + spec, + source, + f"{spec[2].replace('_', ' ').capitalize()} of {instance.label}", + ) + for team, specs in TEAM_REGION_MARTS.items(): + for region in REGIONS: + models = result.regional.get(region.code) + if not models: + continue + folder = f"models/marts/{team}/{region.code}" + for spec in specs: + source = models.get(spec[2]) + if source is not None: + _apply( + project, + f"rpt_{team}_{region.code}__{spec[0]}", + folder, + spec, + source, + f"{spec[2].replace('_', ' ').capitalize()} across {region.name}", + ) + for team, specs in COMBINED_MARTS.items(): + for spec in specs: + source = result.combined.get(spec[2]) + if source is not None: + _apply( + project, + f"rpt_{team}__{spec[0]}", + f"models/marts/{team}", + spec, + source, + f"{spec[2].split('__')[-1].replace('_', ' ').capitalize()} company-wide", + ) + _product_lines(project, result) + published.update(_published(project, result)) + chains.customer_chain(project, result) + chains.finance_chain(project, result) + chains.scorecards(project, result) + _executive(project) + _snapshots(project, result, systems) + return published + + +def _product_lines(project: Project, result: Intermediate) -> None: + for line in PRODUCT_LINES: + source = result.combined.get(f"{line}__daily_demand") + if source is None: + continue + folder = f"models/marts/merchandising/product_lines/{line}" + subject = f"{line.title()} product-line demand" + marts.top_n(project, f"rpt_{line}__top_skus", folder, source, "sku", 50, subject) + marts.share_by( + project, f"rpt_{line}__storefront_share", folder, source, "storefront_code", subject + ) + + +def _published(project: Project, result: Intermediate) -> dict[str, Model]: + folder = "models/marts/core" + combined = result.combined + plan = ( + ("fct_orders", "orders_usd", "order_id", "One row per storefront order with USD amounts"), + ( + "fct_order_lines", + "order_lines_catalogued", + "order_line_id", + "One row per storefront order line with its product and category", + ), + ( + "dim_customer_identities", + "customer_identity", + "storefront_customer_key", + "One row per storefront account with the global customer it belongs to", + ), + ( + "fct_marketplace_orders", + "marketplaces__orders_enriched", + "partner_order_id", + "One row per marketplace order with its items, fees, returns and shipments", + ), + ( + "fct_shipments", + "carriers__shipments_enriched", + "shipment_id", + "One row per carrier shipment with its tracking events and invoice totals", + ), + ( + "fct_store_transactions", + "stores__transactions_enriched", + "txn_id", + "One row per retail store transaction with its lines and tenders", + ), + ( + "fct_purchase_orders", + "suppliers__purchase_orders_enriched", + "po_id", + "One row per supplier purchase order with its invoices", + ), + ( + "fct_gateway_charges", + "payments__charges_enriched", + "charge_id", + "One row per payment gateway charge with its refunds, disputes and fees", + ), + ( + "fct_ad_campaigns", + "advertising__campaigns_enriched", + "campaign_id", + "One row per advertising campaign with its spend and conversions", + ), + ( + "fct_support_tickets", + "support__tickets_enriched", + "ticket_id", + "One row per support ticket with its comments, ratings and events", + ), + ( + "fct_wholesale_orders", + "wholesale__orders_enriched", + "b2b_order_id", + "One row per wholesale order with its lines and invoices", + ), + ( + "fct_return_requests", + "returns__return_requests_enriched", + "return_request_id", + "One row per return request with its labels, inspections and refunds", + ), + ) + published: dict[str, Model] = {} + for name, entity, key, description in plan: + source = combined.get(entity) + if source is not None: + published[name] = marts.publish(project, name, folder, source, key, description) + catalogue = result.per_instance.get("pim") + if catalogue is not None: + published["dim_products"] = marts.publish( + project, + "dim_products", + folder, + catalogue["products_enriched"], + "product_id", + "One row per product with its variant and media counts", + ) + sessions = combined.get("clickstream__sessions") + if sessions is not None: + published["fct_web_sessions"] = marts.publish( + project, + "fct_web_sessions", + folder, + sessions, + "session_id", + "One row per web or app session, merged incrementally with a two-day lookback for " + "sessions that span midnight", + config=[ + "materialized incremental", + "incremental_strategy merge", + "unique_key [session_id]", + "cursor session_start_utc", + "cursor_type timestamp", + "cursor_grain hour", + "lookback 2d", + f"cursor_inputs (\n {sessions.name} session_start_utc,\n )", + "on_schema_change append_new_columns", + ], + ) + return published + + +def _executive(project: Project) -> None: + folder = "models/marts/executive" + daily_sources = [ + (region.code, project.models[f"int_storefronts_{region.code}__orders_enriched_daily"]) + for region in REGIONS + if f"int_storefronts_{region.code}__orders_enriched_daily" in project.models + ] + if len(daily_sources) >= 2: + marts.wide_daily( + project, + "rpt_executive__regional_orders_daily", + folder, + daily_sources, + "One row per day with the order count of every storefront region side by side", + ) + others = [ + ("marketplaces", "int_marketplaces__orders_enriched_daily"), + ("carriers", "int_carriers__shipments_enriched_daily"), + ("warehouses", "int_warehouses__picks_daily"), + ("stores", "int_stores__transactions_enriched_daily"), + ("gateways", "int_payments__charges_enriched_daily"), + ("ledgers", "int_ledgers__gl_entries_daily"), + ] + channel_sources = [ + (alias, project.models[name]) for alias, name in others if name in project.models + ] + channel_sources += daily_sources[:4] + if len(channel_sources) >= 2: + marts.wide_daily( + project, + "rpt_executive__operations_daily", + folder, + channel_sources, + "One row per day with the headline volume of every operating area side by side", + ) + + +def _snapshots( + project: Project, result: Intermediate, systems: dict[str, tuple[Instance, ...]] +) -> None: + for instance in systems.get("storefront", ())[:8]: + customers = result.per_instance[instance.code]["customers"] + tracked = [ + "customer_id", + "email", + "country_code", + "loyalty_tier", + "is_marketing_opt_in", + "source_loaded_at", + ] + _snapshot( + project, + f"snap_{instance.code}__customers", + customers, + "customer_id", + tracked, + f"Version history of {instance.label} customer accounts", + ) + catalogue = result.per_instance.get("pim") + if catalogue is not None: + _snapshot( + project, + "snap_products", + catalogue["products"], + "product_id", + ["product_id", "title", "product_line", "category_path", "source_loaded_at"], + "Version history of products", + ) + _snapshot( + project, + "snap_price_lists", + catalogue["price_lists"], + "price_list_id", + ["price_list_id", "sku", "currency_code", "list_price_amount", "source_loaded_at"], + "Version history of list prices, by checking the price for changes", + check=["list_price_amount"], + ) + + +def _snapshot( + project: Project, + name: str, + source: Model, + key: str, + tracked: list[str], + description: str, + check: list[str] | None = None, +) -> None: + columns = [Column(c, source.column(c).type) for c in tracked] + columns[0] = Column( + key, source.column(key).type, ("not_null (run_scope delta_and_final)",), nullable=False + ) + if check: + config = [ + "materialized snapshot", + f"unique_key [{key}]", + "snapshot_strategy check", + f"check_columns [{', '.join(check)}]", + ] + else: + config = [ + "materialized snapshot", + f"unique_key [{key}]", + "snapshot_strategy timestamp", + "updated_at source_loaded_at", + "invalidate_hard_deletes true", + ] + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte("final", f"SELECT\n{select_list([f's.{c}' for c in tracked])}\nFROM source_rows AS s"), + ] + project.add( + Model( + name=name, + folder="models/snapshots", + layer="snapshots", + description=description, + config=config, + columns=columns, + body=with_query(ctes), + refs=[source.name], + ) + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/marts.py b/benchmarks/duckshop-10k/duckshop10k/marts.py new file mode 100644 index 0000000000..1138cbe548 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/marts.py @@ -0,0 +1,456 @@ +"""The mart layer: per-instance and per-team reports, published facts and dimensions, snapshots.""" + +from __future__ import annotations + +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.staging import with_query + + +def _date_column(model: Model) -> str | None: + return next((c.name for c in model.columns if c.type == "DATE"), None) + + +_NOT_MEASURES = ("_number", "_index", "_rank", "_sequence", "_year", "priority", "position") + + +def _measures(model: Model, limit: int = 3, row_count: bool = False) -> list[str]: + return [ + c.name + for c in model.columns + if c.type in ("NUMBER(18, 2)", "NUMBER(38, 0)", "INT") + and not c.name.endswith(_NOT_MEASURES) + and "unique" not in c.audits + and (row_count or c.name != "row_count") + ][:limit] + + +def _add( + project: Project, + name: str, + folder: str, + description: str, + columns: list[Column], + ctes: list[str], + refs: list[str], + **extra: object, +) -> Model: + return project.add( + Model( + name=name, + folder=folder, + layer="marts", + description=description, + columns=columns, + body=with_query(ctes), + refs=refs, + **extra, # type: ignore[arg-type] + ) + ) + + +def period_rollup( + project: Project, name: str, folder: str, source: Model, grain: str, subject: str +) -> Model | None: + day = _date_column(source) + measures = _measures(source) + if day is None or not measures: + return None + period = f"{grain}_start" + sums = [f"SUM(s.{m})::{_sum_type(source, m)} AS {m}" for m in measures] + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "periods", + "SELECT\n" + + select_list([f"DATE_TRUNC('{grain}', s.{day})::DATE AS {period}", *sums]) + + f"\nFROM source_rows AS s\nWHERE\n s.{day} IS NOT NULL\nGROUP BY\n" + f" DATE_TRUNC('{grain}', s.{day})::DATE", + ), + ] + first = measures[0] + ctes.append(import_cte("calendar", '__ref("dim_date")')) + final = [f"p.{period}", "d.calendar_year", "d.calendar_month", *(f"p.{m}" for m in measures)] + final.append( + f"LAG(p.{first}) OVER (ORDER BY p.{period})::{_sum_type(source, first)} AS previous_{first}" + ) + final.append( + f"@sql.percent_of(p.{first} - LAG(p.{first}) OVER (ORDER BY p.{period})," + f" LAG(p.{first}) OVER (ORDER BY p.{period})) AS {first}_change_percent" + ) + ctes.append( + cte( + "final", + f"SELECT\n{select_list(final)}\nFROM periods AS p\nLEFT JOIN calendar AS d\n" + f" ON p.{period} = d.calendar_date", + ) + ) + columns = [Column(period, "DATE", ("not_null", "unique"), nullable=False)] + columns.extend([Column("calendar_year", "INT"), Column("calendar_month", "INT")]) + columns.extend(Column(m, _sum_type(source, m)) for m in measures) + columns.append(Column(f"previous_{first}", _sum_type(source, first))) + columns.append(Column(f"{first}_change_percent", "NUMBER(9, 4)")) + return _add( + project, + name, + folder, + f"{subject} by {grain}, with the change on the previous {grain}", + columns, + ctes, + [source.name, "dim_date"], + ) + + +def _label(dimension: str) -> str: + """Output name of a dimension published as text; boolean names would imply BOOLEAN.""" + for prefix in ("is_", "has_"): + if dimension.startswith(prefix): + return f"{dimension.removeprefix(prefix)}_label" + return dimension + + +def _sum_type(model: Model, column: str) -> str: + column_type = model.column(column).type + return "INT" if column_type == "INT" else column_type + + +def share_by( + project: Project, name: str, folder: str, source: Model, dimension: str, subject: str +) -> Model | None: + measures = _measures(source, 2) + if not source.has(dimension) or not measures: + return None + first = measures[0] + out = _label(dimension) + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "totals", + "SELECT\n" + + select_list( + [ + f"COALESCE(s.{dimension}::VARCHAR, 'UNKNOWN') AS {out}", + "COUNT(*) AS row_count", + *(f"SUM(s.{m}) AS {m}" for m in measures), + ] + ) + + f"\nFROM source_rows AS s\nGROUP BY\n COALESCE(s.{dimension}::VARCHAR, 'UNKNOWN')", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + f"t.{out}::VARCHAR AS {out}", + "t.row_count::INT AS row_count", + *(f"t.{m}::{_sum_type(source, m)} AS {m}" for m in measures), + f"ROUND(100 * RATIO_TO_REPORT(t.{first}) OVER (), 2)::NUMBER(9, 4)" + f" AS {first}_share_percent", + f"RANK() OVER (ORDER BY t.{first} DESC, t.{out})::INT AS {first}_rank", + ] + ) + + "\nFROM totals AS t", + ), + ] + columns = [ + Column(out, "VARCHAR", ("not_null", "unique"), nullable=False), + Column("row_count", "INT"), + ] + columns.extend(Column(m, _sum_type(source, m)) for m in measures) + columns.extend( + [Column(f"{first}_share_percent", "NUMBER(9, 4)"), Column(f"{first}_rank", "INT")] + ) + return _add( + project, + name, + folder, + f"{subject} by {dimension.replace('_', ' ')}, with each one's share of the total", + columns, + ctes, + [source.name], + ) + + +def top_n( + project: Project, name: str, folder: str, source: Model, entity: str, limit: int, subject: str +) -> Model | None: + measures = _measures(source, 2) + if not source.has(entity) or not measures: + return None + first = measures[0] + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "totals", + "SELECT\n" + + select_list( + [f"s.{entity}", "COUNT(*) AS row_count", *(f"SUM(s.{m}) AS {m}" for m in measures)] + ) + + f"\nFROM source_rows AS s\nWHERE\n s.{entity} IS NOT NULL\nGROUP BY\n s.{entity}", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + f"t.{entity}::VARCHAR AS {entity}", + "t.row_count::INT AS row_count", + *(f"t.{m}::{_sum_type(source, m)} AS {m}" for m in measures), + f"RANK() OVER (ORDER BY t.{first} DESC, t.{entity})::INT AS rank_by_{first}", + ] + ) + + f"\nFROM totals AS t\nQUALIFY RANK() OVER (ORDER BY t.{first} DESC," + f" t.{entity}) <= {limit}", + ), + ] + columns = [ + Column(entity, "VARCHAR", ("not_null", "unique"), nullable=False), + Column("row_count", "INT"), + ] + columns.extend(Column(m, _sum_type(source, m)) for m in measures) + columns.append(Column(f"rank_by_{first}", "INT")) + return _add( + project, + name, + folder, + f"The top {limit} {entity.replace('_', ' ')}s of {subject} by {first.replace('_', ' ')}", + columns, + ctes, + [source.name], + ) + + +def breakdown( + project: Project, name: str, folder: str, source: Model, dimension: str, subject: str +) -> Model | None: + day = _date_column(source) + if not source.has(dimension) or day is None: + return None + label = f"COALESCE(s.{dimension}::VARCHAR, 'UNKNOWN')" + out = _label(dimension) + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "monthly", + "SELECT\n" + + select_list( + [ + f"DATE_TRUNC('month', s.{day})::DATE AS month_start", + f"{label} AS {out}", + "COUNT(*) AS row_count", + ] + ) + + f"\nFROM source_rows AS s\nWHERE\n s.{day} IS NOT NULL\nGROUP BY\n" + f" DATE_TRUNC('month', s.{day})::DATE,\n {label}", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "m.month_start", + f"m.{out}::VARCHAR AS {out}", + "m.row_count::INT AS row_count", + "ROUND(100 * RATIO_TO_REPORT(m.row_count) OVER (PARTITION BY m.month_start), 2)" + "::NUMBER(9, 4) AS share_of_month_percent", + ] + ) + + "\nFROM monthly AS m", + ), + ] + columns = [ + Column("month_start", "DATE", ("not_null",)), + Column(out, "VARCHAR", ("not_null",)), + Column("row_count", "INT"), + Column("share_of_month_percent", "NUMBER(9, 4)"), + ] + return _add( + project, + name, + folder, + f"Monthly count of {subject} by {dimension.replace('_', ' ')} and its share of the month", + columns, + ctes, + [source.name], + ) + + +def cohorts( + project: Project, name: str, folder: str, source: Model, parent: str, moment: str, subject: str +) -> Model | None: + if not (source.has(parent) and source.has(moment)): + return None + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "firsts", + "SELECT\n" + + select_list( + [ + f"s.{parent}", + f"s.{moment}", + f"MIN(s.{moment}) OVER (PARTITION BY s.{parent}) AS first_at", + ] + ) + + f"\nFROM source_rows AS s\nWHERE\n s.{parent} IS NOT NULL", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "DATE_TRUNC('month', f.first_at)::DATE AS cohort_month", + f"DATEDIFF('month', f.first_at, f.{moment})::INT AS months_since_first", + f"COUNT(DISTINCT f.{parent})::INT AS active_count", + "COUNT(*)::INT AS row_count", + ] + ) + + "\nFROM firsts AS f\nGROUP BY\n DATE_TRUNC('month', f.first_at)::DATE,\n" + f" DATEDIFF('month', f.first_at, f.{moment})::INT", + ), + ] + columns = [ + Column("cohort_month", "DATE", ("not_null",)), + Column("months_since_first", "INT", ("non_negative",)), + Column("active_count", "INT"), + Column("row_count", "INT"), + ] + return _add( + project, + name, + folder, + f"Monthly cohorts of {subject}: how many are active each month after their first", + columns, + ctes, + [source.name], + ) + + +REASON_CLASSES: tuple[tuple[str, str], ...] = ( + ("damaged", "(broken|damag|crack|leak)"), + ("wrong_item", "(wrong|incorrect|not what)"), + ("size_fit", "(small|large|size|fit)"), + ("late", "(late|delay|never arrived)"), + ("changed_mind", "(changed my mind|no longer|not needed)"), +) + + +def classify_text( + project: Project, name: str, folder: str, source: Model, text: str, subject: str +) -> Model | None: + if not source.has(text): + return None + cases = "\n".join( + f" WHEN REGEXP_LIKE(LOWER(s.{text}), '.*{pattern}.*') THEN '{label}'" + for label, pattern in REASON_CLASSES + ) + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte( + "classified", + f"SELECT\n CASE\n{cases}\n ELSE 'other'\n END AS reason_class\n" + "FROM source_rows AS s", + ), + cte( + "final", + "SELECT\n" + + select_list( + [ + "c.reason_class::VARCHAR AS reason_class", + "COUNT(*)::INT AS row_count", + "ROUND(100 * RATIO_TO_REPORT(COUNT(*)) OVER (), 2)::NUMBER(9, 4) AS" + " share_percent", + ] + ) + + "\nFROM classified AS c\nGROUP BY\n c.reason_class", + ), + ] + labels = ", ".join(f"'{label}'" for label, _ in REASON_CLASSES) + columns = [ + Column( + "reason_class", + "VARCHAR", + ("not_null", "unique", f"accepted_values (values [{labels}, 'other'])"), + nullable=False, + ), + Column("row_count", "INT"), + Column("share_percent", "NUMBER(9, 4)"), + ] + return _add( + project, + name, + folder, + f"{subject} classified by keywords in their free-text reason", + columns, + ctes, + [source.name], + ) + + +def wide_daily( + project: Project, name: str, folder: str, sources: list[tuple[str, Model]], description: str +) -> Model: + """One row per day joining the first measure of many daily reports, the wide-mart shape.""" + days: list[str] = [] + ctes: list[str] = [] + for alias, source in sources: + day = _date_column(source) + measure = _measures(source, 1, row_count=True)[0] + ctes.append( + cte( + f"{alias}_daily", + "SELECT\n" + + select_list( + [ + f"s.{day}" if day == "report_date" else f"s.{day} AS report_date", + f"SUM(s.{measure}) AS {alias}_{measure}", + ] + ) + + f'\nFROM __ref("{source.name}") AS s\nGROUP BY\n s.{day}', + ) + ) + days.append(f"SELECT d.report_date\nFROM {alias}_daily AS d") + ctes.append(cte("calendar_rows", "\nUNION ALL\n".join(days))) + ctes.append(cte("calendar", "SELECT DISTINCT r.report_date\nFROM calendar_rows AS r")) + items = ["c.report_date"] + joins = [] + columns = [Column("report_date", "DATE", ("not_null", "unique"), nullable=False)] + for alias, source in sources: + measure = _measures(source, 1, row_count=True)[0] + out = f"{alias}_{measure}" + items.append(f"COALESCE({alias}.{out}, 0)::{_sum_type(source, measure)} AS {out}") + joins.append( + f"LEFT JOIN {alias}_daily AS {alias}\n ON c.report_date = {alias}.report_date" + ) + columns.append(Column(out, _sum_type(source, measure))) + ctes.append( + cte("final", f"SELECT\n{select_list(items)}\nFROM calendar AS c\n" + "\n".join(joins)) + ) + return _add(project, name, folder, description, columns, ctes, [s.name for _, s in sources]) + + +def publish( + project: Project, + name: str, + folder: str, + source: Model, + key: str, + description: str, + config: list[str] | None = None, + factory: bool = True, +) -> Model: + """A published fct_ or dim_ model: an enforced contract over an intermediate model.""" + items = [] + columns = [] + for column in source.columns: + items.append(f"s.{column.name}::{column.type} AS {column.name}") + audits = ("not_null", "unique") if column.name == key else () + columns.append(Column(column.name, column.type, audits, column.name != key)) + ctes = [ + import_cte("source_rows", f'__ref("{source.name}")'), + cte("final", f"SELECT\n{select_list(items)}\nFROM source_rows AS s"), + ] + head = list(config or []) + head.append("contract enforced") + if factory and name.startswith("fct_"): + head.append("audit_factories [fact_table_quality]") + return _add(project, name, folder, description, columns, ctes, [source.name], config=head) diff --git a/benchmarks/duckshop-10k/duckshop10k/project.py b/benchmarks/duckshop-10k/duckshop10k/project.py new file mode 100644 index 0000000000..1e93aa0cd5 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/project.py @@ -0,0 +1,216 @@ +"""Assemble the whole DuckShop 10K project in memory, deterministically.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from duckshop10k import GENERATOR_VERSION, PROJECT_NAME, SEED +from duckshop10k.core import Project +from duckshop10k.declarations import currencies_seed, declare_all +from duckshop10k.extras import build_extras +from duckshop10k.layers import build_intermediate +from duckshop10k.mart_layer import build_marts +from duckshop10k.rules_gen import build_rules +from duckshop10k.staging import StagedTable, stage_collector, stage_instance +from duckshop10k.tests_gen import build_scenarios, build_tests +from duckshop10k.world import Instance, instances, storefront_instances + + +@dataclass(frozen=True) +class Profile: + """How many instances of each system to generate. `full` is the published project.""" + + name: str + per_system: int | None + + +PROFILES: dict[str, Profile] = { + "full": Profile("full", None), + "smoke": Profile("smoke", 2), +} + + +@dataclass +class Staging: + tables: dict[str, dict[str, StagedTable]] + events: dict[str, tuple[str, list[str]]] + + def table(self, instance: Instance, table: str) -> StagedTable: + return self.tables[instance.code][table] + + +def _limit(values: tuple[Instance, ...], profile: Profile) -> tuple[Instance, ...]: + return values if profile.per_system is None else values[: profile.per_system] + + +def selected_instances(profile: Profile) -> dict[str, tuple[Instance, ...]]: + systems = {"storefront": storefront_instances(), **instances()} + return {system: _limit(values, profile) for system, values in systems.items()} + + +_STAGING_FOLDER = { + "storefront": "storefronts", + "marketplace": "marketplaces", + "carrier": "carriers", + "warehouse": "warehouses", + "store": "stores", + "supplier": "suppliers", + "gateway": "payments", + "ad_network": "advertising", + "messaging": "messaging", + "affiliate": "affiliates", + "collector": "clickstream", + "support": "support", + "ledger": "ledgers", + "wholesale": "wholesale", + "returns": "returns", + "catalogue": "catalogue", +} + + +def build(profile_name: str = "full") -> dict[str, str]: + profile = PROFILES[profile_name] + project = Project() + declare_all(project) + systems = selected_instances(profile) + staging = Staging(tables={}, events={}) + for system, values in systems.items(): + for instance in values: + folder = f"models/staging/{_STAGING_FOLDER[system]}/{instance.code}" + if system == "collector": + staging.events[instance.code] = stage_collector(project, instance, folder) + else: + staging.tables[instance.code] = { + table.archetype.table: table + for table in stage_instance(project, instance, folder) + } + intermediate = build_intermediate(project, staging.tables, staging.events, systems) + build_marts(project, intermediate, systems) + build_extras(project, intermediate) + build_tests(project, staging.tables, intermediate) + build_scenarios(project, staging.tables) + build_rules(project) + _static_files(project) + project.write("seeds/source_instances.csv", instances_seed(systems)) + return project.render() + + +def _static_files(project: Project) -> None: + project.write("sqlbuild_project.toml", project_toml()) + project.write(".gitignore", "sqlbuild_local.toml\ntarget/\nlogs/\n.sqlbuild/\n") + project.write("seeds/currencies.csv", currencies_seed()) + project.write( + "seeds/seeds.yml", + "seeds:\n" + " - name: currencies\n" + " description: ISO currencies with the number of decimal places in their minor unit\n" + " columns:\n" + " - name: currency_code\n" + " type: VARCHAR\n" + " audits:\n" + " - not_null\n" + " - unique\n" + " - name: minor_unit_exponent\n" + " type: INT\n" + " - name: source_instances\n" + " description: Every deployed source system with its region, time zone and currency\n" + " columns:\n" + " - name: instance_code\n" + " type: VARCHAR\n" + " audits:\n" + " - not_null\n" + " - unique\n" + " - name: system\n" + " type: VARCHAR\n" + " - name: region_code\n" + " type: VARCHAR\n" + " - name: timezone\n" + " type: VARCHAR\n" + " - name: currency_code\n" + " type: VARCHAR\n" + " audits:\n" + " - relationships:\n" + ' to: __seed("currencies")\n' + " field: currency_code\n", + ) + project.write("README.md", readme()) + + +def instances_seed(systems: dict[str, tuple[Instance, ...]]) -> str: + rows = ["instance_code,system,region_code,timezone,currency_code"] + for system, values in systems.items(): + rows.extend( + f"{i.code.upper()},{system},{i.region},{i.timezone},{i.currency}" for i in values + ) + return "\n".join(rows) + "\n" + + +def project_toml() -> str: + return f'''# Generated by DuckShop 10K generator version {GENERATOR_VERSION} (seed {SEED}). +# Do not edit; change the generator in the sqlbuild repository instead. +name = "{PROJECT_NAME}" +adapter = "snowflake" +default_target = "dev" + +# Placeholders only. Compiling needs no connection. +[connections.warehouse] +account = "example-account" +user = "example_user" +role = "EXAMPLE_ROLE" +warehouse = "EXAMPLE_WH" + +[targets.dev] +connection = "warehouse" +database = "DUCKSHOP_10K" +schema = "DEV" + +[settings] +default_audit_severity = "error" +unit_test_max_real_models = 6 + +[defaults] +materialized = "table" + +[path_defaults.staging] +materialized = "view" +tags = ["staging"] + +[path_defaults.intermediate] +materialized = "table" +tags = ["intermediate"] + +[path_defaults.marts] +materialized = "table" +tags = ["marts"] + +[path_defaults.snapshots] +tags = ["snapshots"] + +[vars] +reporting_currency = "USD" +valuation_date = "2026-03-31" +lookup_customer_id = "C000001" + +[rules] +select = ["SQBRSQL", "SQBRGRAPH", "SQBRCONTRACT", "SQBRTEST", "XSQBRDUCK"] +# SQBRSQL021: `SELECT * FROM final` and the change-log macro's pass-through CTEs are the house +# style; each model's columns block, not its terminal SELECT, owns the output shape. +# SQBRCONTRACT101: exact contracts are reserved for published dim_ and fct_ models. +# SQBRTEST202: a sample of models carries unit tests, as in a project of this size. +ignore = ["SQBRSQL021", "SQBRCONTRACT101", "SQBRTEST202"] +''' + + +def readme() -> str: + return f"""# DuckShop 10K + +Generated by the DuckShop 10K generator, version {GENERATOR_VERSION}, seed {SEED}. Every file is +generated: change the generator in the sqlbuild repository (`benchmarks/duckshop-10k/`) and +regenerate instead of editing this project. + +Compile it with no warehouse connection: + +```bash +sqb compile +``` +""" diff --git a/benchmarks/duckshop-10k/duckshop10k/rules_gen.py b/benchmarks/duckshop-10k/duckshop10k/rules_gen.py new file mode 100644 index 0000000000..147748c00a --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/rules_gen.py @@ -0,0 +1,140 @@ +"""The project's custom Rule and its public-harness tests.""" + +from __future__ import annotations + +from duckshop10k.core import Project + +RULE = '''"""DuckShop 10K house rules for the staging layer.""" + +import re + +from sqlbuild.rules import Finding, Model, RuleContext, rule + +_SOURCE_CALL = re.compile(r'__source\\(\\s*"([^"]+)"\\s*\\)') +_DEDUPE = ("@sql.cdc_latest(", "@sql.cdc_history(") + + +def _is_change_log(ctx: RuleContext, source: str) -> bool: + """Whether a source declares an _op column, read from its instance's own sources file.""" + path = f"sources/{source.split('__', 1)[0]}.yml" + if not ctx.project.tree.glob(path): + return False + text = ctx.project.tree.read_text(path) + start = text.find(f" - name: {source}\\n") + if start < 0: + return False + end = text.find("\\n - name: ", start + 1) + block = text[start:] if end < 0 else text[start:end] + return " - name: _op\\n" in block + + +@rule( + code="XSQBRDUCK001", + message="Staging models over a change-log source must deduplicate it with a change-log macro", + remediation=( + "Read the source in an import CTE and follow it with @sql.cdc_latest(, ), or " + "@sql.cdc_history(, ) for a model that keeps every version, so deletes and " + "superseded versions are handled the same way everywhere." + ), +) +def staging_uses_change_log_macro(*, model: Model, ctx: RuleContext) -> list[Finding]: + parts = ctx.project.tree.relative_parts(path=model.path, under="models") + if not parts or parts[0] != "staging": + return [] + sql = ctx.sql.for_model(model).authored.source + if any(macro in sql for macro in _DEDUPE): + return [] + for match in _SOURCE_CALL.finditer(sql): + if _is_change_log(ctx, match.group(1)): + line = sql[: match.start()].count("\\n") + 1 + return [ctx.finding(subject=model, line=line)] + return [] +''' + +RULE_TEST = '''from rules.staging_dedupe import staging_uses_change_log_macro + +from sqlbuild.rules import RuleFile +from sqlbuild.rules.testing import RuleCase, evaluate_rule + +_SOURCES = RuleFile( + path="sources/shop_xx.yml", + source="""sources: + - name: shop_xx__orders + description: Orders change log + columns: + - name: order_id + type: VARCHAR + - name: _op + type: VARCHAR + - name: shop_xx__page_views + description: Append-only page views + columns: + - name: event_id + type: VARCHAR +""", +) + +_RAW_CHANGE_LOG_READ = ( + 'MODEL (description "Orders");\\n' + "WITH src AS (\\n" + ' SELECT * FROM __source("shop_xx__orders")\\n' + "),\\n" + "final AS (\\n" + " SELECT s.order_id FROM src AS s\\n" + ")\\n" + "SELECT * FROM final\\n" +) + + +def test_given_change_log_read_without_dedupe_macro_when_evaluating_then_reports_it() -> None: + result = evaluate_rule( + rule=staging_uses_change_log_macro, + test_case=RuleCase( + description="staging model reads a change log directly", + source=_RAW_CHANGE_LOG_READ, + path="models/staging/storefronts/shop_xx/base_shop_xx__orders.sql", + files=(_SOURCES,), + expected_finding_count=1, + ), + ) + + assert result.finding_count == 1 + assert result.findings[0].line == 3 + + +def test_given_append_only_source_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=staging_uses_change_log_macro, + test_case=RuleCase( + description="append-only sources are deduplicated by delivery sequence instead", + source=_RAW_CHANGE_LOG_READ.replace("shop_xx__orders", "shop_xx__page_views").replace( + "order_id", "event_id" + ), + path="models/staging/storefronts/shop_xx/base_shop_xx__page_views.sql", + files=(_SOURCES,), + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 + + +def test_given_change_log_read_outside_staging_when_evaluating_then_passes() -> None: + result = evaluate_rule( + rule=staging_uses_change_log_macro, + test_case=RuleCase( + description="snapshots and other layers may read the full change log", + source=_RAW_CHANGE_LOG_READ, + path="models/snapshots/snap_orders.sql", + files=(_SOURCES,), + expected_finding_count=0, + ), + ) + + assert result.finding_count == 0 +''' + + +def build_rules(project: Project) -> None: + project.write("rules/staging_dedupe.py", RULE) + project.write("tests/test_rule_staging_dedupe.py", RULE_TEST) diff --git a/benchmarks/duckshop-10k/duckshop10k/specs.py b/benchmarks/duckshop-10k/duckshop10k/specs.py new file mode 100644 index 0000000000..932faeaa0f --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/specs.py @@ -0,0 +1,1681 @@ +"""Source table archetypes for every source system, and how each platform lays them out. + +An archetype lists a table's logical fields by role. A platform generation decides the physical +layout of each field in the raw table (column name, unit, time zone handling, or a JSON path) and +the staging name it is published under: + +- `classic`: legacy names (`_ref`, `_ts`, `_amt`, `_flag`, `status_cd`), local `TIMESTAMP_NTZ`, + major-unit amounts, and JSON kept as text in `extra_json`; +- `nest`: conventional columns, `TIMESTAMP_TZ`, minor-unit amounts, JSON in `attributes`; +- `flock`: a key plus one `doc` document holding every other field, with epoch-millisecond times. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +# Roles whose raw value is semi-structured; they always live in the JSON payload. +SEMI_ROLES = frozenset({"array", "object"}) + + +@dataclass(frozen=True) +class Field: + stem: str + role: str + in_payload: bool = False + + +@dataclass(frozen=True) +class Child: + """An array inside a parent row, published as its own staging model by LATERAL FLATTEN.""" + + name: str + array: str + fields: tuple[Field, ...] + description: str + + +@dataclass(frozen=True) +class Archetype: + table: str + key: str + fields: tuple[Field, ...] + description: str + append: bool = False + children: tuple[Child, ...] = () + history: bool = False + + +def f(stem: str, role: str, payload: bool = False) -> Field: + return Field(stem, role, payload or role in SEMI_ROLES) + + +# Output names and types in staging. Times and amounts keep their source unit in staging, so the +# name says which one it is; the intermediate conformed models convert them. +def staging_name(field: Field, platform: str) -> str: + if field.role == "ts": + if field.in_payload and platform != "flock": + return f"{field.stem}_at" + return { + "classic": f"{field.stem}_at_local", + "nest": f"{field.stem}_at", + "flock": f"{field.stem}_at_utc", + }[platform] + if field.role == "amount": + if platform == "classic" and not field.in_payload: + return f"{field.stem}_amount" + return f"{field.stem}_amount_minor" + if field.role == "status": + return "status_code" if platform == "classic" else "status" + if field.role == "date": + return f"{field.stem}_date" + if field.role == "bool": + return f"is_{field.stem}" + if field.role == "url": + return f"{field.stem}_path" + return field.stem + + +def staging_type(field: Field, platform: str) -> str: + role = field.role + if role == "ts": + if field.in_payload and platform != "flock": + return "TIMESTAMP_TZ" + return {"classic": "TIMESTAMP_NTZ", "nest": "TIMESTAMP_TZ", "flock": "TIMESTAMP_NTZ"}[ + platform + ] + if role == "amount": + if platform == "classic" and not field.in_payload: + return "NUMBER(18, 2)" + return "NUMBER(38, 0)" + if role == "status": + return "INT" if platform == "classic" else "VARCHAR" + return { + "qty": "INT", + "int": "INT", + "rate": "NUMBER(18, 6)", + "date": "DATE", + "bool": "BOOLEAN", + "array": "ARRAY", + "object": "OBJECT", + }.get(role, "VARCHAR") + + +def payload_column(platform: str) -> str: + return {"classic": "extra", "nest": "attributes", "flock": "doc"}[platform] + + +def lives_in_payload(field: Field, platform: str) -> bool: + return platform == "flock" or field.in_payload + + +def physical_name(field: Field, platform: str) -> str: + """Raw column name of a field that is not in the payload.""" + stem, role = field.stem, field.role + if platform == "classic": + if role in ("id", "key") and stem.endswith("_id"): + return stem[: -len("_id")] + "_ref" + if role == "ts": + return f"{stem}_ts" + if role == "amount": + return f"{stem}_amt" + if role == "bool": + return f"{stem}_flag" + if role == "status": + return "status_cd" + if role == "date": + return f"{stem}_dt" + return stem + if role == "ts": + return f"{stem}_at" + if role == "amount": + return f"{stem}_amount_minor" + if role == "bool": + return f"is_{stem}" + if role == "date": + return f"{stem}_date" + return stem + + +def physical_type(field: Field, platform: str) -> str: + role = field.role + if platform == "classic": + return { + "ts": "TIMESTAMP_NTZ", + "amount": "NUMBER(18,2)", + "bool": "VARCHAR", + "status": "NUMBER(38,0)", + "qty": "NUMBER(38,0)", + "int": "NUMBER(38,0)", + "rate": "NUMBER(18,6)", + "date": "DATE", + }.get(role, "VARCHAR") + return { + "ts": "TIMESTAMP_TZ", + "amount": "NUMBER(38,0)", + "bool": "BOOLEAN", + "qty": "NUMBER(38,0)", + "int": "NUMBER(38,0)", + "rate": "NUMBER(18,6)", + "date": "DATE", + }.get(role, "VARCHAR") + + +def json_path(field: Field, platform: str) -> str: + """Path of a payload field inside the platform's JSON document.""" + if platform == "flock": + if field.role == "ts": + return f"{field.stem}_ms" + if field.role == "amount": + return f"amounts.{field.stem}" + return field.stem + + +# --- Archetypes ------------------------------------------------------------------------------- + +_STOREFRONT: tuple[Archetype, ...] = ( + Archetype( + "customers", + "customer_id", + ( + f("email", "email"), + f("phone", "phone"), + f("full_name", "name"), + f("country_code", "country"), + f("created", "ts"), + f("marketing_opt_in", "bool", True), + f("preferred_language", "code", True), + f("birth_year", "int", True), + f("loyalty_tier", "code", True), + f("consents", "array"), + ), + "customer accounts", + children=(Child("consents", "consents", (f("consent_type", "code"),), "consent grants"),), + history=True, + ), + Archetype( + "customer_addresses", + "address_id", + ( + f("customer_id", "id"), + f("line1", "text"), + f("city", "name"), + f("postcode", "postcode"), + f("country_code", "country"), + f("default", "bool"), + f("valid_from", "ts"), + ), + "customer address book entries", + history=True, + ), + Archetype( + "orders", + "order_id", + ( + f("customer_id", "id"), + f("currency_code", "currency"), + f("placed", "ts"), + f("status", "status"), + f("gross", "amount"), + f("discount", "amount"), + f("tax", "amount"), + f("shipping", "amount"), + f("channel", "code", True), + f("utm_source", "code", True), + f("utm_campaign", "code", True), + f("device_type", "code", True), + f("gift", "bool", True), + f("line_skus", "array"), + ), + "orders", + history=True, + ), + Archetype( + "order_lines", + "order_line_id", + ( + f("order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("unit_price", "amount"), + f("discounts", "array"), + ), + "order lines", + children=( + Child( + "discounts", + "discounts", + (f("promo_code", "code"), f("discount", "amount", True), f("kind", "code")), + "discounts applied to order lines", + ), + ), + ), + Archetype( + "payments", + "payment_id", + ( + f("order_id", "id"), + f("method", "code"), + f("captured", "amount"), + f("currency_code", "currency"), + f("authorised", "ts"), + f("gateway_status", "code", True), + f("risk_score", "rate", True), + f("three_ds", "code", True), + f("attempts", "array"), + ), + "payments", + children=( + Child( + "attempts", + "attempts", + (f("attempt_number", "int"), f("result", "code"), f("error_code", "code")), + "payment authorisation attempts", + ), + ), + ), + Archetype( + "refunds", + "refund_id", + ( + f("payment_id", "id"), + f("refunded", "amount"), + f("reason_text", "text"), + f("issued", "ts"), + f("initiated_by", "code", True), + ), + "refunds", + ), + Archetype( + "carts", + "cart_id", + ( + f("customer_id", "id"), + f("anonymous_id", "id"), + f("opened", "ts"), + f("last_activity", "ts"), + f("items", "array"), + ), + "shopping carts", + children=( + Child( + "items", + "items", + (f("sku", "sku"), f("quantity", "qty"), f("unit_price", "amount", True)), + "cart items", + ), + ), + ), + Archetype( + "wishlists", + "wishlist_item_id", + (f("customer_id", "id"), f("sku", "sku"), f("added", "ts"), f("priority", "int", True)), + "wishlist items", + ), + Archetype( + "reviews", + "review_id", + ( + f("sku", "sku"), + f("customer_id", "id"), + f("rating", "int"), + f("title", "text"), + f("submitted", "ts"), + f("body_length", "int", True), + f("moderation", "code", True), + ), + "product reviews", + ), + Archetype( + "subscriptions", + "subscription_id", + ( + f("customer_id", "id"), + f("plan_code", "code"), + f("started", "ts"), + f("status", "status"), + f("price", "amount"), + f("currency_code", "currency"), + f("billing_interval", "code", True), + ), + "subscription boxes", + history=True, + ), + Archetype( + "loyalty_events", + "loyalty_event_id", + ( + f("customer_id", "id"), + f("event_type", "code"), + f("points", "int"), + f("occurred", "ts"), + f("tier_after", "code", True), + ), + "loyalty point events", + ), + Archetype( + "gift_card_redemptions", + "redemption_id", + ( + f("gift_card_id", "id"), + f("order_id", "id"), + f("redeemed", "amount"), + f("currency_code", "currency"), + f("used", "ts"), + ), + "gift card redemptions", + ), + Archetype( + "consent_events", + "consent_event_id", + ( + f("customer_id", "id"), + f("consent_type", "code"), + f("granted", "bool"), + f("occurred", "ts"), + f("source_channel", "code", True), + ), + "consent changes", + ), + Archetype( + "promotion_redemptions", + "promotion_redemption_id", + ( + f("promo_code", "code"), + f("order_id", "id"), + f("customer_id", "id"), + f("discount", "amount"), + f("redeemed", "ts"), + f("tiers", "array"), + ), + "promotion redemptions", + children=( + Child( + "tiers", + "tiers", + (f("threshold", "amount", True), f("discount_rate", "rate")), + "promotion tiers reached", + ), + ), + ), + Archetype( + "tax_lines", + "tax_line_id", + ( + f("order_id", "id"), + f("jurisdiction", "code"), + f("tax_rate", "rate"), + f("tax", "amount"), + ), + "order tax lines", + ), + Archetype( + "store_credit", + "credit_txn_id", + ( + f("customer_id", "id"), + f("credit", "amount"), + f("currency_code", "currency"), + f("reason", "code"), + f("occurred", "ts"), + ), + "store credit transactions", + ), + Archetype( + "order_notes", + "note_id", + (f("order_id", "id"), f("author_type", "code"), f("body", "text"), f("written", "ts")), + "order notes", + ), + Archetype( + "newsletter_signups", + "signup_id", + ( + f("email", "email"), + f("country_code", "country"), + f("signed_up", "ts"), + f("source_page", "url", True), + f("double_opt_in", "bool", True), + ), + "newsletter sign-ups", + ), +) + +_MARKETPLACE: tuple[Archetype, ...] = ( + Archetype( + "listings", + "listing_id", + ( + f("sku", "sku"), + f("title", "text"), + f("listing_status", "code"), + f("price", "amount"), + f("currency_code", "currency"), + f("updated", "ts"), + f("images", "array"), + f("attributes_bag", "object"), + ), + "partner listings", + children=( + Child("images", "images", (f("image_url", "url"), f("position", "int")), "images"), + ), + history=True, + ), + Archetype( + "orders", + "partner_order_id", + ( + f("buyer_ref", "id"), + f("placed", "ts"), + f("currency_code", "currency"), + f("total", "amount"), + f("status", "status"), + f("ship_country", "country"), + ), + "partner orders", + history=True, + ), + Archetype( + "order_items", + "partner_order_item_id", + ( + f("partner_order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("item_price", "amount"), + f("fees", "array"), + ), + "partner order items", + children=( + Child("fees", "fees", (f("fee_type", "code"), f("fee", "amount", True)), "item fees"), + ), + ), + Archetype( + "returns", + "partner_return_id", + ( + f("partner_order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("reason_text", "text"), + f("received", "ts"), + ), + "partner returns", + ), + Archetype( + "payouts", + "payout_id", + ( + f("period_start", "date"), + f("period_end", "date"), + f("payout", "amount"), + f("currency_code", "currency"), + f("paid", "ts"), + ), + "partner payouts", + ), + Archetype( + "fees", + "fee_id", + ( + f("partner_order_id", "id"), + f("fee_type", "code"), + f("fee", "amount"), + f("currency_code", "currency"), + f("charged", "ts"), + ), + "partner fees", + ), + Archetype( + "inventory_snapshots", + "inventory_snapshot_id", + (f("sku", "sku"), f("available", "qty"), f("reserved", "qty"), f("captured", "ts")), + "partner stock snapshots", + append=True, + ), + Archetype( + "messages", + "message_id", + ( + f("partner_order_id", "id"), + f("direction", "code"), + f("body", "text"), + f("sent", "ts"), + ), + "buyer messages", + append=True, + ), + Archetype( + "shipments", + "partner_shipment_id", + ( + f("partner_order_id", "id"), + f("carrier_code", "code"), + f("tracking_number", "id"), + f("shipped", "ts"), + f("delivered", "ts"), + ), + "partner-fulfilled shipments", + ), + Archetype( + "reviews", + "partner_review_id", + ( + f("sku", "sku"), + f("rating", "int"), + f("title", "text"), + f("submitted", "ts"), + f("verified", "bool", True), + ), + "partner product reviews", + ), + Archetype( + "promotions", + "partner_promotion_id", + ( + f("sku", "sku"), + f("promotion_type", "code"), + f("starts", "ts"), + f("ends", "ts"), + f("deal_price", "amount"), + ), + "partner promotions", + ), + Archetype( + "disputes", + "partner_dispute_id", + ( + f("partner_order_id", "id"), + f("dispute_type", "code"), + f("opened", "ts"), + f("claimed", "amount"), + f("status", "status"), + ), + "buyer disputes", + ), +) + +_CARRIER: tuple[Archetype, ...] = ( + Archetype( + "shipments", + "shipment_id", + ( + f("order_id", "id"), + f("warehouse_code", "code"), + f("service_level", "code"), + f("shipped", "ts"), + f("weight_grams", "int"), + f("tracking", "array"), + ), + "shipments", + children=( + Child( + "tracking", + "tracking", + (f("event_code", "code"), f("location", "name"), f("sequence", "int")), + "embedded tracking events", + ), + ), + ), + Archetype( + "tracking_events", + "tracking_event_id", + ( + f("shipment_id", "id"), + f("status_code_text", "code"), + f("occurred", "ts"), + f("location", "name"), + ), + "tracking feed events", + append=True, + ), + Archetype( + "invoices", + "invoice_line_id", + ( + f("shipment_id", "id"), + f("charge_type", "code"), + f("charge", "amount"), + f("currency_code", "currency"), + f("invoiced", "ts"), + ), + "carrier invoice lines", + ), + Archetype( + "rate_cards", + "rate_card_id", + ( + f("service_level", "code"), + f("zone", "code"), + f("weight_band", "int"), + f("rate", "amount"), + f("currency_code", "currency"), + f("valid_from", "ts"), + ), + "carrier rate cards", + history=True, + ), +) + +_WAREHOUSE: tuple[Archetype, ...] = ( + Archetype( + "receipts", + "receipt_id", + (f("po_id", "id"), f("sku", "sku"), f("quantity", "qty"), f("received", "ts")), + "goods receipts", + ), + Archetype( + "picks", + "pick_id", + ( + f("order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("picked", "ts"), + f("picker_ref", "id"), + ), + "pick tasks", + ), + Archetype( + "packs", + "pack_id", + ( + f("order_id", "id"), + f("package_count", "int"), + f("packed", "ts"), + f("dimensions", "object"), + ), + "packed orders", + ), + Archetype( + "stock_counts", + "count_id", + (f("sku", "sku"), f("counted_qty", "qty"), f("counted", "ts")), + "stock counts", + ), + Archetype( + "adjustments", + "adjustment_id", + (f("sku", "sku"), f("qty_delta", "int"), f("reason_code", "code"), f("adjusted", "ts")), + "stock adjustments", + ), + Archetype( + "labour_shifts", + "shift_id", + ( + f("team_code", "code"), + f("started", "ts"), + f("ended", "ts"), + f("headcount", "int"), + ), + "labour shifts", + ), + Archetype( + "dock_appointments", + "appointment_id", + ( + f("carrier_code", "code"), + f("scheduled", "ts"), + f("arrived", "ts"), + f("door", "code"), + ), + "dock appointments", + ), + Archetype( + "stock_movements", + "movement_id", + (f("sku", "sku"), f("qty_delta", "int"), f("reason_code", "code"), f("moved", "ts")), + "stock movements", + append=True, + ), + Archetype( + "putaways", + "putaway_id", + ( + f("receipt_id", "id"), + f("sku", "sku"), + f("bin_code", "code"), + f("quantity", "qty"), + f("completed", "ts"), + ), + "put-away tasks", + ), + Archetype( + "replenishments", + "replenishment_id", + ( + f("sku", "sku"), + f("from_bin", "code"), + f("to_bin", "code"), + f("quantity", "qty"), + f("requested", "ts"), + f("completed", "ts"), + ), + "pick-face replenishments", + ), + Archetype( + "cycle_counts", + "cycle_count_id", + ( + f("bin_code", "code"), + f("sku", "sku"), + f("expected_qty", "qty"), + f("counted_qty", "qty"), + f("counted", "ts"), + ), + "cycle counts", + ), + Archetype( + "damages", + "damage_id", + ( + f("sku", "sku"), + f("quantity", "qty"), + f("damage_type", "code"), + f("recorded", "ts"), + f("photos", "array"), + ), + "damaged stock records", + ), + Archetype( + "returns_received", + "returned_parcel_id", + ( + f("order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("condition_code", "code"), + f("received", "ts"), + ), + "customer returns received", + ), + Archetype( + "slotting", + "slot_id", + ( + f("bin_code", "code"), + f("sku", "sku"), + f("zone", "code"), + f("assigned", "ts"), + ), + "bin slotting assignments", + history=True, + ), +) + +_STORE: tuple[Archetype, ...] = ( + Archetype( + "transactions", + "txn_id", + ( + f("register_id", "code"), + f("opened", "ts"), + f("closed", "ts"), + f("total", "amount"), + f("currency_code", "currency"), + f("status", "status"), + ), + "till transactions", + ), + Archetype( + "transaction_lines", + "txn_line_id", + ( + f("txn_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("unit_price", "amount"), + f("discounts", "array"), + ), + "till transaction lines", + children=( + Child( + "discounts", + "discounts", + (f("promo_code", "code"), f("discount", "amount", True)), + "till line discounts", + ), + ), + ), + Archetype( + "tenders", + "tender_id", + ( + f("txn_id", "id"), + f("tender_type", "code"), + f("tendered", "amount"), + f("card_bin", "code"), + ), + "transaction tenders", + ), + Archetype( + "shifts", + "till_shift_id", + ( + f("register_id", "code"), + f("opened", "ts"), + f("closed", "ts"), + f("float", "amount"), + ), + "till shifts", + ), + Archetype( + "footfall", + "footfall_id", + (f("counted", "ts"), f("visitors", "int"), f("sensor", "code")), + "door counter readings", + append=True, + ), + Archetype( + "returns", + "store_return_id", + ( + f("txn_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("refunded", "amount"), + f("reason_code", "code"), + f("processed", "ts"), + ), + "in-store returns", + ), + Archetype( + "gift_cards", + "store_gift_card_id", + ( + f("txn_id", "id"), + f("loaded", "amount"), + f("card_suffix", "code"), + f("issued", "ts"), + ), + "gift cards sold in store", + ), + Archetype( + "cash_drops", + "cash_drop_id", + ( + f("register_id", "code"), + f("dropped", "amount"), + f("dropped_at_shift", "id"), + f("recorded", "ts"), + ), + "till cash drops", + ), + Archetype( + "price_overrides", + "price_override_id", + ( + f("txn_line_id", "id"), + f("original_price", "amount"), + f("override_price", "amount"), + f("reason_code", "code"), + f("approved", "ts"), + ), + "manual price overrides", + ), + Archetype( + "loyalty_scans", + "loyalty_scan_id", + ( + f("txn_id", "id"), + f("member_ref", "id"), + f("scanned", "ts"), + f("points_earned", "int"), + ), + "loyalty card scans at the till", + append=True, + ), + Archetype( + "staff_discounts", + "staff_discount_id", + ( + f("txn_id", "id"), + f("discount", "amount"), + f("approved_by_ref", "id"), + f("granted", "ts"), + ), + "staff discounts", + ), +) + +_SUPPLIER: tuple[Archetype, ...] = ( + Archetype( + "purchase_orders", + "po_id", + ( + f("warehouse_code", "code"), + f("ordered", "ts"), + f("expected", "ts"), + f("currency_code", "currency"), + f("lines", "array"), + ), + "purchase orders", + children=( + Child( + "lines", + "lines", + (f("sku", "sku"), f("quantity", "qty"), f("unit_cost", "amount", True)), + "purchase order lines", + ), + ), + ), + Archetype( + "asn", + "asn_id", + (f("po_id", "id"), f("dispatched", "ts"), f("eta", "ts"), f("cartons", "array")), + "advance shipping notices", + children=( + Child( + "cartons", + "cartons", + (f("carton_ref", "id"), f("sku", "sku"), f("quantity", "qty")), + "cartons on a shipping notice", + ), + ), + ), + Archetype( + "invoices", + "supplier_invoice_id", + ( + f("po_id", "id"), + f("invoiced_total", "amount"), + f("currency_code", "currency"), + f("invoiced", "ts"), + f("due", "date"), + ), + "supplier invoices", + ), + Archetype( + "catalogue_items", + "supplier_sku_id", + ( + f("sku", "sku"), + f("unit_cost", "amount"), + f("currency_code", "currency"), + f("lead_time_days", "int"), + f("valid_from", "ts"), + ), + "supplier catalogue items", + history=True, + ), +) + +_GATEWAY: tuple[Archetype, ...] = ( + Archetype( + "charges", + "charge_id", + ( + f("order_id", "id"), + f("charged", "amount"), + f("currency_code", "currency"), + f("created", "ts"), + f("status", "status"), + f("card_bin", "code", True), + f("risk", "object"), + ), + "gateway charges", + ), + Archetype( + "refunds", + "gateway_refund_id", + (f("charge_id", "id"), f("refunded", "amount"), f("created", "ts"), f("reason", "code")), + "gateway refunds", + ), + Archetype( + "disputes", + "dispute_id", + ( + f("charge_id", "id"), + f("disputed", "amount"), + f("opened", "ts"), + f("reason_code", "code"), + f("evidence", "array"), + ), + "card disputes", + children=( + Child( + "evidence", + "evidence", + (f("document_type", "code"), f("page_count", "int")), + "dispute evidence documents", + ), + ), + ), + Archetype( + "payouts", + "gateway_payout_id", + (f("payout", "amount"), f("currency_code", "currency"), f("arrival", "date")), + "gateway payouts", + ), + Archetype( + "fees", + "gateway_fee_id", + (f("charge_id", "id"), f("fee_type", "code"), f("fee", "amount")), + "gateway fees", + ), +) + +_AD_NETWORK: tuple[Archetype, ...] = ( + Archetype( + "campaigns", + "campaign_id", + ( + f("campaign_name", "text"), + f("objective", "code"), + f("started", "ts"), + f("ended", "ts"), + f("budget", "amount"), + f("currency_code", "currency"), + ), + "ad campaigns", + ), + Archetype( + "ad_groups", + "ad_group_id", + (f("campaign_id", "id"), f("ad_group_name", "text"), f("bid_strategy", "code")), + "ad groups", + ), + Archetype( + "daily_spend", + "spend_id", + ( + f("campaign_id", "id"), + f("spend", "date"), + f("spent", "amount"), + f("currency_code", "currency"), + f("metrics", "array"), + ), + "daily spend reports", + append=True, + children=( + Child( + "metrics", + "metrics", + (f("metric", "code"), f("metric_value", "rate")), + "reported delivery metrics", + ), + ), + ), + Archetype( + "conversions", + "conversion_id", + ( + f("campaign_id", "id"), + f("click_id", "id"), + f("order_id", "id"), + f("converted", "ts"), + f("conversion", "amount"), + f("currency_code", "currency"), + ), + "reported conversions", + append=True, + ), + Archetype( + "audiences", + "audience_id", + (f("audience_name", "text"), f("audience_size", "int"), f("refreshed", "ts")), + "audience lists", + ), +) + +_MESSAGING: tuple[Archetype, ...] = ( + Archetype( + "sends", + "send_id", + (f("campaign_id", "id"), f("recipient_email", "email"), f("sent", "ts")), + "message sends", + append=True, + ), + Archetype( + "opens", + "open_id", + (f("send_id", "id"), f("opened", "ts"), f("client_text", "text")), + "message opens", + append=True, + ), + Archetype( + "clicks", + "click_id", + (f("send_id", "id"), f("link", "url"), f("clicked", "ts")), + "message link clicks", + append=True, + ), + Archetype( + "bounces", + "bounce_id", + (f("send_id", "id"), f("bounce_type", "code"), f("bounced", "ts")), + "message bounces", + append=True, + ), + Archetype( + "unsubscribes", + "unsubscribe_id", + (f("recipient_email", "email"), f("unsubscribed", "ts"), f("reason", "code")), + "unsubscribes", + append=True, + ), +) + +_AFFILIATE: tuple[Archetype, ...] = ( + Archetype( + "clicks", + "affiliate_click_id", + (f("affiliate_id", "id"), f("landing", "url"), f("clicked", "ts")), + "affiliate clicks", + append=True, + ), + Archetype( + "conversions", + "affiliate_conversion_id", + ( + f("affiliate_click_id", "id"), + f("order_id", "id"), + f("commission", "amount"), + f("currency_code", "currency"), + f("converted", "ts"), + f("status", "status"), + ), + "affiliate conversions", + ), + Archetype( + "payouts", + "affiliate_payout_id", + ( + f("affiliate_id", "id"), + f("payout", "amount"), + f("currency_code", "currency"), + f("paid", "ts"), + ), + "affiliate payouts", + ), +) + +_SUPPORT: tuple[Archetype, ...] = ( + Archetype( + "tickets", + "ticket_id", + ( + f("customer_email", "email"), + f("contact_channel", "code"), + f("priority", "int"), + f("created", "ts"), + f("custom_fields", "object"), + f("tags", "array"), + ), + "support tickets", + children=(Child("tags", "tags", (f("tag", "code"),), "ticket tags"),), + ), + Archetype( + "ticket_comments", + "comment_id", + ( + f("ticket_id", "id"), + f("author_type", "code"), + f("body", "text"), + f("written", "ts"), + ), + "ticket comments", + ), + Archetype( + "csat_responses", + "csat_id", + (f("ticket_id", "id"), f("score", "int"), f("comment", "text"), f("responded", "ts")), + "satisfaction survey responses", + ), + Archetype( + "ticket_events", + "ticket_event_id", + ( + f("ticket_id", "id"), + f("event_type", "code"), + f("occurred", "ts"), + f("details", "object"), + ), + "ticket lifecycle events", + append=True, + ), +) + +_LEDGER: tuple[Archetype, ...] = ( + Archetype( + "gl_entries", + "gl_entry_id", + ( + f("account_code", "code"), + f("cost_centre", "code"), + f("posted_value", "amount"), + f("currency_code", "currency"), + f("posted", "ts"), + f("journal_ref", "id"), + ), + "general ledger entries", + ), + Archetype( + "gl_accounts", + "account_code", + (f("account_name", "text"), f("account_type", "code"), f("active", "bool")), + "chart of accounts", + ), + Archetype( + "ap_invoices", + "ap_invoice_id", + ( + f("supplier_ref", "id"), + f("invoiced_total", "amount"), + f("currency_code", "currency"), + f("invoiced", "ts"), + f("due", "date"), + f("status", "status"), + ), + "accounts payable invoices", + ), + Archetype( + "ar_invoices", + "ar_invoice_id", + ( + f("customer_ref", "id"), + f("invoiced_total", "amount"), + f("currency_code", "currency"), + f("invoiced", "ts"), + f("due", "date"), + f("status", "status"), + ), + "accounts receivable invoices", + ), + Archetype( + "budgets", + "budget_line_id", + ( + f("cost_centre", "code"), + f("period", "date"), + f("budgeted", "amount"), + f("currency_code", "currency"), + ), + "budget lines", + ), + Archetype( + "bank_transactions", + "bank_txn_id", + ( + f("booked_value", "amount"), + f("currency_code", "currency"), + f("booked", "date"), + f("counterparty", "text"), + f("reference", "text"), + ), + "bank statement lines", + append=True, + ), + Archetype( + "journals", + "journal_id", + ( + f("journal_type", "code"), + f("posted", "ts"), + f("period", "date"), + f("description_text", "text"), + f("reversed", "bool"), + ), + "ledger journals", + ), + Archetype( + "cost_centres", + "cost_centre_code", + ( + f("cost_centre_name", "text"), + f("parent_code", "code"), + f("owner_team", "code"), + f("active", "bool"), + ), + "cost centres", + history=True, + ), + Archetype( + "fixed_assets", + "asset_id", + ( + f("asset_class", "code"), + f("acquired", "ts"), + f("cost", "amount"), + f("currency_code", "currency"), + f("useful_life_months", "int"), + ), + "fixed asset register", + ), + Archetype( + "depreciation_runs", + "depreciation_id", + ( + f("asset_id", "id"), + f("period", "date"), + f("depreciation", "amount"), + f("currency_code", "currency"), + ), + "monthly depreciation", + ), + Archetype( + "tax_codes", + "tax_code", + ( + f("tax_rate", "rate"), + f("jurisdiction", "code"), + f("valid_from", "ts"), + ), + "tax codes", + history=True, + ), + Archetype( + "vendors", + "vendor_id", + ( + f("vendor_name", "name"), + f("country_code", "country"), + f("payment_terms", "code"), + f("email", "email"), + ), + "vendor master", + history=True, + ), + Archetype( + "intercompany", + "intercompany_id", + ( + f("counterparty_entity", "code"), + f("settled", "amount"), + f("currency_code", "currency"), + f("posted", "ts"), + ), + "intercompany settlements", + ), + Archetype( + "accruals", + "accrual_id", + ( + f("account_code", "code"), + f("accrued", "amount"), + f("currency_code", "currency"), + f("period", "date"), + f("reversed", "bool"), + ), + "month-end accruals", + ), + Archetype( + "bank_accounts", + "bank_account_id", + ( + f("bank_name", "name"), + f("currency_code", "currency"), + f("iban_suffix", "code"), + f("opened", "ts"), + ), + "bank accounts", + ), + Archetype( + "periods", + "period_id", + ( + f("period_start", "date"), + f("period_end", "date"), + f("closed", "bool"), + f("closed_on", "ts"), + ), + "accounting periods", + ), +) + +_WHOLESALE: tuple[Archetype, ...] = ( + Archetype( + "accounts", + "account_id", + ( + f("company_name", "name"), + f("country_code", "country"), + f("credit_limit", "amount"), + f("currency_code", "currency"), + f("opened", "ts"), + ), + "wholesale accounts", + history=True, + ), + Archetype( + "price_agreements", + "agreement_id", + ( + f("account_id", "id"), + f("sku", "sku"), + f("net_price", "amount"), + f("valid_from", "ts"), + f("valid_to", "ts"), + ), + "negotiated price agreements", + ), + Archetype( + "orders", + "b2b_order_id", + ( + f("account_id", "id"), + f("placed", "ts"), + f("currency_code", "currency"), + f("total", "amount"), + f("status", "status"), + f("po_reference", "text"), + ), + "wholesale orders", + ), + Archetype( + "order_lines", + "b2b_order_line_id", + ( + f("b2b_order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("net_price", "amount"), + ), + "wholesale order lines", + ), + Archetype( + "invoices", + "b2b_invoice_id", + ( + f("b2b_order_id", "id"), + f("invoiced_total", "amount"), + f("currency_code", "currency"), + f("invoiced", "ts"), + f("due", "date"), + ), + "wholesale invoices", + ), + Archetype( + "payments", + "b2b_payment_id", + ( + f("b2b_invoice_id", "id"), + f("received_value", "amount"), + f("received", "ts"), + f("method", "code"), + ), + "wholesale payments", + ), + Archetype( + "credit_notes", + "credit_note_id", + ( + f("b2b_invoice_id", "id"), + f("credited", "amount"), + f("issued", "ts"), + f("reason", "code"), + ), + "wholesale credit notes", + ), + Archetype( + "contacts", + "contact_id", + (f("account_id", "id"), f("email", "email"), f("phone", "phone"), f("role_code", "code")), + "wholesale account contacts", + ), +) + +_RETURNS: tuple[Archetype, ...] = ( + Archetype( + "return_requests", + "return_request_id", + ( + f("order_id", "id"), + f("sku", "sku"), + f("quantity", "qty"), + f("reason_text", "text"), + f("requested", "ts"), + f("photos", "array"), + ), + "return requests", + ), + Archetype( + "return_labels", + "label_id", + (f("return_request_id", "id"), f("carrier_code", "code"), f("created", "ts")), + "return labels", + ), + Archetype( + "inspections", + "inspection_id", + ( + f("return_request_id", "id"), + f("grade", "code"), + f("inspected", "ts"), + f("notes", "text"), + ), + "returned item inspections", + ), + Archetype( + "refunds_issued", + "refund_issue_id", + ( + f("return_request_id", "id"), + f("refunded", "amount"), + f("currency_code", "currency"), + f("issued", "ts"), + ), + "refunds issued for returns", + ), +) + +_CATALOGUE: tuple[Archetype, ...] = ( + Archetype( + "products", + "product_id", + ( + f("title", "text"), + f("category_path", "text"), + f("product_line", "code"), + f("launched", "ts"), + f("specs", "object"), + ), + "products", + history=True, + ), + Archetype( + "product_variants", + "sku", + ( + f("product_id", "id"), + f("colour", "code"), + f("size_label", "code"), + f("weight_grams", "int"), + f("barcodes", "array"), + ), + "product variants", + children=(Child("barcodes", "barcodes", (f("barcode", "code"),), "variant barcodes"),), + history=True, + ), + Archetype( + "categories", + "category_id", + (f("parent_category_id", "id"), f("slug", "code"), f("display_name", "name")), + "catalogue categories", + ), + Archetype( + "price_lists", + "price_list_id", + ( + f("sku", "sku"), + f("currency_code", "currency"), + f("list_price", "amount"), + f("valid_from", "ts"), + f("valid_to", "ts"), + ), + "list prices", + history=True, + ), + Archetype( + "bundles", + "bundle_id", + (f("title", "text"), f("components", "array")), + "product bundles", + children=( + Child( + "components", + "components", + (f("sku", "sku"), f("quantity", "qty")), + "bundle components", + ), + ), + ), + Archetype( + "media", + "media_id", + (f("product_id", "id"), f("asset", "url"), f("position", "int")), + "product media", + ), +) + +ARCHETYPES: dict[str, tuple[Archetype, ...]] = { + "storefront": _STOREFRONT, + "marketplace": _MARKETPLACE, + "carrier": _CARRIER, + "warehouse": _WAREHOUSE, + "store": _STORE, + "supplier": _SUPPLIER, + "gateway": _GATEWAY, + "ad_network": _AD_NETWORK, + "messaging": _MESSAGING, + "affiliate": _AFFILIATE, + "support": _SUPPORT, + "ledger": _LEDGER, + "wholesale": _WHOLESALE, + "returns": _RETURNS, + "catalogue": _CATALOGUE, +} + +# Clickstream collectors deliver one append-only event table; staging splits it by event type. +EVENT_TYPES: tuple[tuple[str, tuple[Field, ...]], ...] = ( + ("page_view", (f("title", "text", True), f("referrer", "url", True))), + ("product_view", (f("sku", "sku", True), f("list_name", "code", True))), + ("add_to_cart", (f("sku", "sku", True), f("quantity", "qty", True))), + ("remove_from_cart", (f("sku", "sku", True), f("quantity", "qty", True))), + ("cart_view", (f("item_count", "int", True),)), + ("checkout_step", (f("step", "int", True), f("option", "code", True))), + ("shipping_info", (f("shipping_tier", "code", True),)), + ("payment_info", (f("payment_type", "code", True),)), + ("checkout_complete", (f("order_id", "id", True), f("revenue", "amount", True))), + ("search", (f("search_term", "text", True), f("result_count", "int", True))), + ("filter_applied", (f("filter_name", "code", True), f("filter_value", "text", True))), + ("sort_changed", (f("sort_order", "code", True),)), + ("wishlist_add", (f("sku", "sku", True),)), + ("review_open", (f("sku", "sku", True), f("rating_filter", "int", True))), + ("video_play", (f("video_id", "id", True), f("position_seconds", "int", True))), + ("promo_click", (f("promo_id", "id", True), f("creative", "code", True))), + ("coupon_applied", (f("promo_code", "code", True),)), + ("coupon_failed", (f("promo_code", "code", True), f("failure_reason", "code", True))), + ("login", (f("method", "code", True),)), + ("signup", (f("method", "code", True), f("newsletter", "bool", True))), + ("consent_banner", (f("choice", "code", True),)), + ("store_locator", (f("postcode", "postcode", True), f("result_count", "int", True))), + ("size_guide", (f("sku", "sku", True),)), + ("share", (f("sku", "sku", True), f("share_target", "code", True))), + ("chat_open", (f("entry_point", "code", True),)), + ("recommendation_click", (f("sku", "sku", True), f("slot", "int", True))), + ("scroll_depth", (f("depth_percent", "int", True),)), + ("outbound_link", (f("target", "url", True),)), + ("error_shown", (f("error_code", "code", True), f("message", "text", True))), + ("gift_message", (f("message_length", "int", True),)), +) diff --git a/benchmarks/duckshop-10k/duckshop10k/staging.py b/benchmarks/duckshop-10k/duckshop10k/staging.py new file mode 100644 index 0000000000..418d80dc67 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/staging.py @@ -0,0 +1,738 @@ +"""Sources and the staging layer: base_ change-log dedupe, typed stg_ models and their fan-outs.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from duckshop10k.core import Column, Model, Project, cte, import_cte, select_list +from duckshop10k.specs import ( + ARCHETYPES, + EVENT_TYPES, + Archetype, + Child, + Field, + json_path, + lives_in_payload, + payload_column, + physical_name, + physical_type, + staging_name, + staging_type, +) +from duckshop10k.world import Instance + +RAW_DATABASE = "DUCKSHOP_RAW" + + +@dataclass(frozen=True) +class StagedTable: + """What later layers need to know about one staged source table.""" + + instance: Instance + archetype: Archetype + stg: str + children: dict[str, str] + history: str | None + + +def model_type(source_type: str) -> str: + return source_type.replace(",", ", ") + + +def with_query(parts: list[str]) -> str: + """WITH list in house style; CTE-list macro calls are spliced in without a comma.""" + out = "WITH " + for index, part in enumerate(parts): + out += part + if index == len(parts) - 1: + break + out += "\n\n" if part.startswith("@sql.") else ",\n\n" + return out + "\n\nSELECT *\nFROM final\n" + + +# --- Field transforms ------------------------------------------------------------------------- + + +def json_transform(role: str, value: str, platform: str) -> str: + text = f"{value}::VARCHAR" + return { + "key": f"REGEXP_REPLACE(@sql.json_text({value}), '[^A-Za-z0-9_:.-]', '')::VARCHAR", + "id": f"@sql.json_text({value})", + "code": f"UPPER(@sql.json_text({value}))::VARCHAR", + "email": f"@sql.normalise_email({text})", + "phone": f"@sql.normalise_phone({text})", + "sku": f"@sql.normalise_sku({text})", + "text": f"NULLIF(TRIM(REGEXP_REPLACE({text}, '\\\\s+', ' ')), '')::VARCHAR", + "name": f"INITCAP(TRIM({text}))::VARCHAR", + "country": f"UPPER(TRIM({text}))::VARCHAR", + "currency": f"UPPER(TRIM({text}))::VARCHAR", + "postcode": f"UPPER(REGEXP_REPLACE({text}, '[^A-Za-z0-9]', ''))::VARCHAR", + "url": f"('/' || LTRIM(PARSE_URL({text}, 1)['path']::VARCHAR, '/'))::VARCHAR", + "amount": f"TRY_TO_NUMBER({text})::NUMBER(38, 0)", + "qty": f"TRY_TO_NUMBER({text})::INT", + "int": f"TRY_TO_NUMBER({text})::INT", + "rate": f"TRY_TO_DECIMAL({text}, 18, 6)::NUMBER(18, 6)", + "ts": ( + f"@sql.epoch_ms_to_utc(TRY_TO_NUMBER({text})::NUMBER(38, 0))" + if platform == "flock" + else f"TRY_TO_TIMESTAMP_TZ({text})" + ), + "date": f"TRY_TO_DATE({text})", + "bool": f"COALESCE({value}::BOOLEAN, FALSE)", + "status": f"LOWER(TRIM({text}))::VARCHAR", + "array": f"COALESCE({value}, ARRAY_CONSTRUCT())::ARRAY", + "object": f"COALESCE({value}, OBJECT_CONSTRUCT())::OBJECT", + }[role] + + +def column_transform(role: str, value: str, platform: str) -> str: + classic = platform == "classic" + return { + "key": f"REGEXP_REPLACE(TRIM({value}), '[^A-Za-z0-9_:.-]', '')::VARCHAR", + "id": f"TRIM({value})::VARCHAR", + "code": f"UPPER(TRIM({value}))::VARCHAR", + "email": f"@sql.normalise_email({value})", + "phone": f"@sql.normalise_phone({value})", + "sku": f"@sql.normalise_sku({value})", + "text": f"NULLIF(TRIM(REGEXP_REPLACE({value}, '\\\\s+', ' ')), '')::VARCHAR", + "name": f"INITCAP(TRIM({value}))::VARCHAR", + "country": f"UPPER(TRIM({value}))::VARCHAR", + "currency": f"UPPER(TRIM({value}))::VARCHAR", + "postcode": f"UPPER(REGEXP_REPLACE({value}, '[^A-Za-z0-9]', ''))::VARCHAR", + "url": f"('/' || LTRIM(PARSE_URL({value}, 1)['path']::VARCHAR, '/'))::VARCHAR", + "amount": (f"ROUND({value}, 2)::NUMBER(18, 2)" if classic else f"{value}::NUMBER(38, 0)"), + "qty": f"{value}::INT", + "int": f"{value}::INT", + "rate": f"{value}::NUMBER(18, 6)", + "ts": f"{value}::TIMESTAMP_NTZ" if classic else f"{value}::TIMESTAMP_TZ", + "date": f"{value}::DATE", + "bool": ( + f"COALESCE(UPPER({value}) = 'Y', FALSE)::BOOLEAN" + if classic + else f"COALESCE({value}, FALSE)::BOOLEAN" + ), + "status": f"{value}::INT" if classic else f"LOWER(TRIM({value}))::VARCHAR", + }[role] + + +def key_field(archetype: Archetype) -> Field: + return Field(archetype.key, "key") + + +# --- Sources ---------------------------------------------------------------------------------- + + +def physical_columns(archetype: Archetype, platform: str) -> list[tuple[str, str]]: + """(name, type) of every raw column, excluding the change-log metadata.""" + key = key_field(archetype) + columns = [(physical_name(key, platform), "VARCHAR")] + if platform == "flock": + return [*columns, ("doc", "VARIANT")] + for item in archetype.fields: + if not lives_in_payload(item, platform): + columns.append((physical_name(item, platform), physical_type(item, platform))) + if any(item.in_payload for item in archetype.fields): + payload = "extra_json" if platform == "classic" else "attributes" + columns.append((payload, "VARCHAR" if platform == "classic" else "VARIANT")) + return columns + + +def source_yaml(instance: Instance, archetype: Archetype) -> str: + name = f"{instance.code}__{archetype.table}" + lines = [ + f" - name: {name}", + f" description: {archetype.description.capitalize()} from {instance.label}" + + (", append-only" if archetype.append else ", change log"), + f" database: {RAW_DATABASE}", + f" schema: {instance.code.upper()}", + f" table: {archetype.table.upper()}", + " contract: enforced", + " columns:", + ] + for index, (column, column_type) in enumerate(physical_columns(archetype, instance.platform)): + lines.append(f" - name: {column}") + lines.append(f" type: {column_type}") + if index == 0: + lines.extend([" nullable: false", " audits:", " - not_null"]) + if not archetype.append: + lines.extend( + [ + " - name: _op", + " type: VARCHAR", + " nullable: false", + " audits:", + " - accepted_values:", + ' values: ["I", "U", "D"]', + ] + ) + lines.extend( + [ + " - name: _seq", + " type: NUMBER(38,0)", + " nullable: false", + " audits:", + " - not_null", + " - unique", + " - name: _loaded_at", + " type: TIMESTAMP_NTZ", + " nullable: false", + " - name: _meta", + " type: VARIANT", + ] + ) + return "\n".join(lines) + + +# --- base_ models ----------------------------------------------------------------------------- + + +def base_model(project: Project, instance: Instance, archetype: Archetype, folder: str) -> Model: + source = f"{instance.code}__{archetype.table}" + platform = instance.platform + raw = physical_columns(archetype, platform) + key = raw[0][0] + ctes = [import_cte("src", f'__source("{source}")')] + if archetype.append: + ctes.append( + cte( + "deduplicated", + "SELECT *\nFROM src AS s\n" + f"QUALIFY ROW_NUMBER() OVER (PARTITION BY s.{key} ORDER BY s._seq DESC) = 1", + ) + ) + relation = "deduplicated" + else: + ctes.append(f"@sql.cdc_latest(src, {key})") + relation = "cdc_latest" + items: list[str] = [] + columns: list[Column] = [] + for index, (name, column_type) in enumerate(raw): + if name == "extra_json": + items.append("TRY_PARSE_JSON(c.extra_json) AS extra") + columns.append(Column("extra", "VARIANT")) + continue + items.append(f"c.{name}") + audits = ("not_null", "unique") if index == 0 else () + columns.append(Column(name, model_type(column_type), audits, nullable=index != 0)) + items.extend( + [ + "c._seq", + "c._loaded_at", + "TRY_TO_NUMBER(c._meta:schema_version::VARCHAR)::INT AS _schema_version", + "COALESCE(c._meta, OBJECT_CONSTRUCT())::OBJECT AS _meta", + ] + ) + columns.extend( + [ + Column("_seq", "NUMBER(38, 0)"), + Column("_loaded_at", "TIMESTAMP_NTZ"), + Column("_schema_version", "INT"), + Column("_meta", "OBJECT"), + ] + ) + ctes.append(cte("final", f"SELECT\n{select_list(items)}\nFROM {relation} AS c")) + kind = "append-only feed" if archetype.append else "change log" + return project.add( + Model( + name=f"base_{source}", + folder=folder, + layer="staging", + description=( + f"Latest version of each row of the {archetype.description} {kind} from " + f"{instance.label}" + ), + columns=columns, + body=with_query(ctes), + sources=[source], + ) + ) + + +# --- stg_ models ------------------------------------------------------------------------------ + + +def field_expression(item: Field, platform: str, alias: str = "b") -> str: + if lives_in_payload(item, platform): + value = f"{alias}.{payload_column(platform)}:{json_path(item, platform)}" + return json_transform(item.role, value, platform) + return column_transform(item.role, f"{alias}.{physical_name(item, platform)}", platform) + + +def _sequence_window(archetype: Archetype, platform: str) -> tuple[str, str] | None: + """A per-parent sequence number, the row-numbering idiom staging uses for ordering.""" + if platform == "flock": + return None + parent = next((item for item in archetype.fields if item.role == "id"), None) + moment = next( + (item for item in archetype.fields if item.role == "ts" and not item.in_payload), None + ) + if parent is None or moment is None: + return None + key = physical_name(key_field(archetype), platform) + name = f"{parent.stem.removesuffix('_id')}_{archetype.table.removesuffix('s')}_number" + expression = ( + f"ROW_NUMBER() OVER (\n PARTITION BY b.{physical_name(parent, platform)}\n" + f" ORDER BY b.{physical_name(moment, platform)}, b.{key}\n )::INT AS {name}" + ) + return name, expression + + +def stg_model(project: Project, instance: Instance, archetype: Archetype, folder: str) -> Model: + platform = instance.platform + source = f"{instance.code}__{archetype.table}" + key = key_field(archetype) + items = [ + f"{column_transform('key', f'b.{physical_name(key, platform)}', platform)} AS {key.stem}" + ] + columns = [Column(key.stem, "VARCHAR", ("not_null", "unique"), nullable=False)] + for item in archetype.fields: + name = staging_name(item, platform) + items.append(f"{field_expression(item, platform)} AS {name}") + audits: tuple[str, ...] = () + if item.role == "currency": + audits = ("valid_currency",) + elif item.role == "qty": + audits = ("non_negative",) + columns.append(Column(name, staging_type(item, platform), audits)) + window = _sequence_window(archetype, platform) + if window is not None: + items.append(window[1]) + columns.append(Column(window[0], "INT")) + items.extend( + [ + "b._schema_version AS source_schema_version", + "@sql.json_text(b._meta:origin.txid) AS source_txid", + "COALESCE(b._meta:origin.backfill::BOOLEAN, FALSE) AS is_backfill_row", + "@sql.load_metadata(b)", + ] + ) + columns.extend( + [ + Column("source_schema_version", "INT"), + Column("source_txid", "VARCHAR"), + Column("is_backfill_row", "BOOLEAN"), + Column("source_seq", "NUMBER(38, 0)"), + Column("source_loaded_at", "TIMESTAMP_NTZ"), + ] + ) + ctes = [ + import_cte("base", f'__ref("base_{source}")'), + cte("final", f"SELECT\n{select_list(items)}\nFROM base AS b"), + ] + return project.add( + Model( + name=f"stg_{source}", + folder=folder, + layer="staging", + description=( + f"One row per live {archetype.description.removesuffix('s')} record from " + f"{instance.label}, typed and normalised from the {platform} platform layout" + ), + columns=columns, + body=with_query(ctes), + refs=[f"base_{source}"], + ) + ) + + +def child_model( + project: Project, instance: Instance, archetype: Archetype, child: Child, folder: str +) -> Model: + parent = f"stg_{instance.code}__{archetype.table}" + platform = instance.platform + key = archetype.key + child_key = f"{child.name}_key" + items = [ + f"(p.{key} || '-' || e.index::VARCHAR)::VARCHAR AS {child_key}", + f"p.{key}", + "e.index::INT AS element_index", + ] + columns = [ + Column(child_key, "VARCHAR", ("not_null", "unique"), nullable=False), + Column(key, "VARCHAR", ("not_null",)), + Column("element_index", "INT"), + ] + for item in child.fields: + payload_item = Field(item.stem, item.role, True) + name = staging_name(payload_item, "nest") + items.append(f"{json_transform(item.role, f'e.value:{item.stem}', 'nest')} AS {name}") + columns.append(Column(name, staging_type(payload_item, "nest"))) + array = staging_name(next(x for x in archetype.fields if x.stem == child.array), platform) + ctes = [ + import_cte("parent", f'__ref("{parent}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM parent AS p, LATERAL FLATTEN(input =>" + f" p.{array}) AS e", + ), + ] + return project.add( + Model( + name=f"{parent}__{child.name}", + folder=folder, + layer="staging", + description=f"One row per element of the {child.description} array on {parent}", + columns=columns, + body=with_query(ctes), + refs=[parent], + ) + ) + + +def generic_child_model( + project: Project, instance: Instance, archetype: Archetype, item: Field, folder: str +) -> Model: + """Array elements as text, or object entries as key and value pairs, one row each.""" + parent = f"stg_{instance.code}__{archetype.table}" + key = archetype.key + child_key = f"{item.stem}_key" + column = staging_name(item, instance.platform) + if item.role == "array": + position = "e.index" + detail = [ + "e.index::INT AS element_index", + "@sql.json_text(e.value) AS element_value", + "TYPEOF(e.value)::VARCHAR AS element_type", + ] + detail_columns = [ + Column("element_index", "INT"), + Column("element_value", "VARCHAR"), + Column("element_type", "VARCHAR"), + ] + noun = "element" + else: + position = "e.key" + detail = [ + "LOWER(e.key)::VARCHAR AS attribute_key", + "@sql.json_text(e.value) AS attribute_value", + "IS_OBJECT(e.value)::BOOLEAN AS is_nested", + ] + detail_columns = [ + Column("attribute_key", "VARCHAR"), + Column("attribute_value", "VARCHAR"), + Column("is_nested", "BOOLEAN"), + ] + noun = "entry" + items = [ + f"(p.{key} || '-' || {position}::VARCHAR)::VARCHAR AS {child_key}", + f"p.{key}", + *detail, + ] + columns = [ + Column(child_key, "VARCHAR", ("not_null", "unique"), nullable=False), + Column(key, "VARCHAR", ("not_null",)), + *detail_columns, + ] + ctes = [ + import_cte("parent", f'__ref("{parent}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM parent AS p, LATERAL FLATTEN(input =>" + f" p.{column}) AS e", + ), + ] + return project.add( + Model( + name=f"{parent}__{item.stem}", + folder=folder, + layer="staging", + description=( + f"One row per {noun} of {item.stem.replace('_', ' ')} on {parent}, flattened " + "from the semi-structured column" + ), + columns=columns, + body=with_query(ctes), + refs=[parent], + ) + ) + + +def history_model(project: Project, instance: Instance, archetype: Archetype, folder: str) -> Model: + """Every version of each key with its validity window, for slowly changing reports.""" + source = f"{instance.code}__{archetype.table}" + platform = instance.platform + key_physical = physical_name(key_field(archetype), platform) + tracked = [item for item in archetype.fields if item.role in ("code", "status", "amount")][:3] + items = [ + f"{column_transform('key', f'h.{key_physical}', platform)} AS {archetype.key}", + "h._seq::NUMBER(38, 0) AS version_seq", + '(h._op = @enum("cdc_op").DELETE)::BOOLEAN AS is_deleted', + "h._loaded_at::TIMESTAMP_NTZ AS valid_from_loaded_at", + "h.next_loaded_at::TIMESTAMP_NTZ AS valid_to_loaded_at", + "(h.next_loaded_at IS NULL)::BOOLEAN AS is_current", + ] + columns = [ + Column(archetype.key, "VARCHAR", ("not_null",)), + Column("version_seq", "NUMBER(38, 0)", ("not_null", "unique"), nullable=False), + Column("is_deleted", "BOOLEAN"), + Column("valid_from_loaded_at", "TIMESTAMP_NTZ"), + Column("valid_to_loaded_at", "TIMESTAMP_NTZ"), + Column("is_current", "BOOLEAN"), + ] + for item in tracked: + name = staging_name(item, platform) + items.append(f"{field_expression(item, platform, 'h')} AS {name}") + columns.append(Column(name, staging_type(item, platform))) + needs_payload = platform == "classic" and any(item.in_payload for item in tracked) + versions = ["s.*", "TRY_PARSE_JSON(s.extra_json) AS extra"] if needs_payload else ["s.*"] + versions.append( + f"LEAD(s._loaded_at) OVER (PARTITION BY s.{key_physical} ORDER BY s._seq) AS next_loaded_at" + ) + ctes = [ + import_cte("src", f'__source("{source}")'), + f"@sql.cdc_history(src, {key_physical})", + cte("versions", f"SELECT\n{select_list(versions)}\nFROM cdc_history AS s"), + cte("final", f"SELECT\n{select_list(items)}\nFROM versions AS h"), + ] + return project.add( + Model( + name=f"stg_{source}__history", + folder=folder, + layer="staging", + description=( + f"Every version of each {archetype.description.removesuffix('s')} record from " + f"{instance.label}, with the load times it was valid between" + ), + columns=columns, + body=with_query(ctes), + sources=[source], + ) + ) + + +def stage_instance(project: Project, instance: Instance, folder: str) -> list[StagedTable]: + yaml_entries: list[str] = [] + staged: list[StagedTable] = [] + for archetype in ARCHETYPES[instance.system]: + yaml_entries.append(source_yaml(instance, archetype)) + base_model(project, instance, archetype, folder) + stg = stg_model(project, instance, archetype, folder) + children = { + child.name: child_model(project, instance, archetype, child, folder).name + for child in archetype.children + } + for item in archetype.fields: + if item.role in ("array", "object") and item.stem not in { + child.array for child in archetype.children + }: + children[item.stem] = generic_child_model( + project, instance, archetype, item, folder + ).name + history = ( + history_model(project, instance, archetype, folder).name + if archetype.history and not archetype.append + else None + ) + staged.append(StagedTable(instance, archetype, stg.name, children, history)) + project.write( + f"sources/{instance.code}.yml", + f"# Raw tables from {instance.label}.\nsources:\n" + "\n".join(yaml_entries) + "\n", + ) + return staged + + +# --- Clickstream ------------------------------------------------------------------------------ + +EVENT_COLUMNS: tuple[tuple[str, str], ...] = ( + ("event_id", "VARCHAR"), + ("received_at", "TIMESTAMP_NTZ"), + ("event_type", "VARCHAR"), + ("event_at_utc", "TIMESTAMP_NTZ"), + ("anonymous_id", "VARCHAR"), + ("customer_id", "VARCHAR"), + ("storefront_code", "VARCHAR"), + ("page_path", "VARCHAR"), + ("referrer_host", "VARCHAR"), + ("client_timezone", "VARCHAR"), + ("is_bot", "BOOLEAN"), + ("properties", "OBJECT"), + ("items", "ARRAY"), +) + + +def stage_collector(project: Project, instance: Instance, folder: str) -> tuple[str, list[str]]: + source = f"{instance.code}__events" + project.write( + f"sources/{instance.code}.yml", + "\n".join( + [ + f"# Raw events from the {instance.label} collector.", + "sources:", + f" - name: {source}", + f" description: Tracker events received by the {instance.label} collector", + f" database: {RAW_DATABASE}", + f" schema: {instance.code.upper()}", + " table: EVENTS", + " contract: enforced", + " columns:", + " - name: event_id", + " type: VARCHAR", + " nullable: false", + " audits:", + " - not_null", + " - name: received_at", + " type: TIMESTAMP_NTZ", + " nullable: false", + " - name: event", + " type: VARIANT", + " - name: _seq", + " type: NUMBER(38,0)", + " nullable: false", + " audits:", + " - not_null", + " - unique", + " - name: _loaded_at", + " type: TIMESTAMP_NTZ", + " nullable: false", + "", + ] + ), + ) + base = project.add( + Model( + name=f"base_{source}", + folder=folder, + layer="staging", + description=f"Each tracker event from the {instance.label} collector once", + columns=[ + Column("event_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("received_at", "TIMESTAMP_NTZ", ("not_null",)), + Column("event", "VARIANT"), + Column("_seq", "NUMBER(38, 0)"), + Column("_loaded_at", "TIMESTAMP_NTZ"), + ], + body=with_query( + [ + import_cte("src", f'__source("{source}")'), + cte( + "final", + "SELECT\n s.event_id,\n s.received_at,\n s.event,\n s._seq,\n" + " s._loaded_at\nFROM src AS s\n" + "QUALIFY ROW_NUMBER() OVER (PARTITION BY s.event_id ORDER BY" + " s._seq DESC) = 1", + ), + ] + ), + sources=[source], + ) + ) + items = [ + "b.event_id", + "b.received_at", + "LOWER(REGEXP_REPLACE(b.event:type::VARCHAR, '[^A-Za-z]+', '_'))::VARCHAR AS event_type", + "COALESCE(\n @sql.epoch_ms_to_utc(b.event:ts_ms::NUMBER(38, 0)),\n b.received_at\n )" + "::TIMESTAMP_NTZ AS event_at_utc", + "LOWER(b.event:anonymous_id::VARCHAR)::VARCHAR AS anonymous_id", + "@sql.json_text(b.event:user.id) AS customer_id", + "UPPER(b.event:context.storefront::VARCHAR)::VARCHAR AS storefront_code", + "('/' || LTRIM(PARSE_URL(b.event:page.url::VARCHAR, 1)['path']::VARCHAR, '/'))" + "::VARCHAR AS page_path", + "LOWER(PARSE_URL(b.event:page.referrer::VARCHAR, 1)['host']::VARCHAR)::VARCHAR" + " AS referrer_host", + "b.event:context.tz::VARCHAR AS client_timezone", + "COALESCE(REGEXP_LIKE(\n b.event:context.ua::VARCHAR,\n " + ' @const("bot_user_agent_pattern"),' + "\n 'i'\n ), FALSE)::BOOLEAN AS is_bot", + "COALESCE(b.event:properties, OBJECT_CONSTRUCT())::OBJECT AS properties", + "COALESCE(b.event:items, ARRAY_CONSTRUCT())::ARRAY AS items", + ] + columns = [Column(name, column_type) for name, column_type in EVENT_COLUMNS] + columns[0] = Column( + "event_id", "VARCHAR", ("not_null (run_scope delta_and_final)", "unique"), nullable=False + ) + columns[1] = Column("received_at", "TIMESTAMP_NTZ", ("not_null",)) + events = project.add( + Model( + name=f"stg_{source}", + folder=folder, + layer="staging", + description=( + f"One row per tracker event from the {instance.label} collector, typed" + " from the raw " + "payload and built in six-hour microbatches. An event without a client timestamp " + "takes the time the collector received it." + ), + config=[ + "materialized incremental", + "incremental_strategy delete_insert", + "incremental_mode microbatch", + "microbatch_strategy watermark", + "cursor_watermark_mode all", + "cursor received_at", + "cursor_type timestamp", + "cursor_grain hour", + "batch_size 6h", + "lookback 2h", + "replay_on_change full", + "cursor_inputs (\n " + f"{base.name} (column received_at, roles [filter, watermark]),\n )", + "on_schema_change append_new_columns", + ], + columns=columns, + body=with_query( + [ + import_cte("base", f'__ref("{base.name}")'), + cte("final", f"SELECT\n{select_list(items)}\nFROM base AS b"), + ] + ), + refs=[base.name], + ) + ) + typed: list[str] = [] + for event_type, fields in EVENT_TYPES: + typed.append(event_model(project, instance, events, event_type, fields, folder).name) + return events.name, typed + + +def event_model( + project: Project, + instance: Instance, + events: Model, + event_type: str, + fields: tuple[Field, ...], + folder: str, +) -> Model: + items = [ + "e.event_id", + "e.event_at_utc", + "e.anonymous_id", + "e.customer_id", + "e.storefront_code", + "e.page_path", + ] + columns = [ + Column("event_id", "VARCHAR", ("not_null", "unique"), nullable=False), + Column("event_at_utc", "TIMESTAMP_NTZ", ("not_null",)), + Column("anonymous_id", "VARCHAR"), + Column("customer_id", "VARCHAR"), + Column("storefront_code", "VARCHAR"), + Column("page_path", "VARCHAR"), + ] + for item in fields: + name = staging_name(item, "nest") + items.append(f"{json_transform(item.role, f'e.properties:{item.stem}', 'nest')} AS {name}") + columns.append(Column(name, staging_type(item, "nest"))) + body = with_query( + [ + import_cte("events", f'__ref("{events.name}")'), + cte( + "final", + f"SELECT\n{select_list(items)}\nFROM events AS e\n" + f"WHERE\n e.event_type = '{event_type}'\n AND NOT e.is_bot", + ), + ] + ) + return project.add( + Model( + name=f"stg_{instance.code}__{event_type}_events", + folder=folder, + layer="staging", + description=( + f"Human {event_type.replace('_', ' ')} events from the {instance.label} collector " + "with their typed properties" + ), + columns=columns, + body=body, + refs=[events.name], + ) + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/stats.py b/benchmarks/duckshop-10k/duckshop10k/stats.py new file mode 100644 index 0000000000..76d0c62cf0 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/stats.py @@ -0,0 +1,70 @@ +"""Shape statistics of a generated project, read back from its files.""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import dataclass +from functools import cache + +_REF = re.compile(r'__ref\("([a-z0-9_]+)"\)') +_SOURCE = re.compile(r'__source\("([a-z0-9_]+)"\)') +_TABLE_FN = re.compile(r'__table_fn\("([a-z0-9_]+)"\)') +_TABLE_FN_READS = {"table_fn__customer_orders": "fct_orders"} + +LAYERS = ("staging", "intermediate", "marts", "snapshots") + + +@dataclass(frozen=True) +class Shape: + models: int + layers: dict[str, int] + sources: int + unit_tests: int + scenarios: int + upstream_counts: Counter[int] + depth: Counter[int] + + @property + def max_depth(self) -> int: + return max(self.depth) + + def layer_percent(self, layer: str) -> float: + return 100 * self.layers.get(layer, 0) / self.models + + +def _models(files: dict[str, str]) -> dict[str, tuple[str, str]]: + return { + path.rsplit("/", 1)[1].removesuffix(".sql"): (path.split("/")[1], text) + for path, text in files.items() + if path.startswith("models/") and "/_sqlbuild/" not in path and path.endswith(".sql") + } + + +def shape(files: dict[str, str]) -> Shape: + models = _models(files) + upstream: dict[str, set[str]] = {} + for name, (_layer, text) in models.items(): + reads = set(_REF.findall(text)) + reads |= {_TABLE_FN_READS[fn] for fn in _TABLE_FN.findall(text)} + upstream[name] = reads + sources = {source for _layer, text in models.values() for source in _SOURCE.findall(text)} + + @cache + def depth(name: str) -> int: + parents = upstream[name] + return 1 + max((depth(parent) for parent in parents), default=0) + + return Shape( + models=len(models), + layers=dict(Counter(layer for layer, _text in models.values())), + sources=len(sources), + unit_tests=sum( + text.count("TEST (") + for path, text in files.items() + if path.startswith("tests/unit/") and path.endswith(".sql") + ), + scenarios=sum(1 for path in files if path.startswith("tests/scenarios/")), + upstream_counts=Counter(len(parents) for parents in upstream.values()), + depth=Counter(depth(name) for name in models), + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/tests_gen.py b/benchmarks/duckshop-10k/duckshop10k/tests_gen.py new file mode 100644 index 0000000000..3054acc197 --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/tests_gen.py @@ -0,0 +1,664 @@ +"""A sampled set of unit tests, a few scenarios, and the custom rule's test, all generated.""" + +from __future__ import annotations + +from duckshop10k.core import Model, Project +from duckshop10k.layers import Intermediate +from duckshop10k.specs import Field, physical_name +from duckshop10k.staging import StagedTable, physical_columns +from duckshop10k.world import CURRENCY_EXPONENT + +# One in this many staging dedupe models, conformed models and so on gets a unit test. +SAMPLE_STRIDE = 20 + + +def _null(column_type: str) -> str: + return f"CAST(NULL AS {column_type})" + + +def _rows(columns: list[tuple[str, str]], rows: list[dict[str, str]]) -> str: + """UNION ALL of literal rows; columns missing from a row are typed NULLs.""" + selects = [] + for index, row in enumerate(rows): + values = [] + for name, column_type in columns: + value = row.get(name, _null(column_type)) + values.append(f" {value} AS {name}" if index == 0 else f" {value}") + selects.append(" SELECT\n" + ",\n".join(values)) + return "\n UNION ALL\n".join(selects) + + +def _expected(names: list[str], rows: list[list[str]]) -> str: + """Expected rows; every projection is aliased in every branch, as tests require.""" + return "\n UNION ALL\n".join( + " SELECT\n" + + ",\n".join(f" {value} AS {name}" for name, value in zip(names, row, strict=True)) + for row in rows + ) + + +def _test_path(model: Model, behaviour: str) -> str: + folder = model.folder.removeprefix("models/") + return f"tests/unit/{folder}/test_{model.name}__{behaviour}.sql" + + +def _write_test( + project: Project, model: Model, behaviour: str, fixtures: list[tuple[str, str]], extra: str = "" +) -> None: + blocks = [f"{name} AS (\n{body}\n)" for name, body in fixtures] + text = f'TEST (name "{model.name}__{behaviour}");\n\nWITH ' + ",\n\n".join(blocks) + project.write(_test_path(model, behaviour), text + extra + "\n") + + +def _instances_fixture(staged: StagedTable) -> tuple[str, str]: + instance = staged.instance + return ( + "__seed__source_instances", + f" SELECT\n '{instance.code.upper()}' AS instance_code,\n" + f" '{instance.system}' AS system,\n '{instance.region}' AS region_code,\n" + f" '{instance.timezone}' AS timezone,\n '{instance.currency}' AS currency_code", + ) + + +def _model_columns(model: Model) -> list[tuple[str, str]]: + return [(column.name, column.type) for column in model.columns] + + +def base_dedupe_test(project: Project, staged: StagedTable) -> None: + instance, archetype = staged.instance, staged.archetype + source = f"{instance.code}__{archetype.table}" + model = project.ref(f"base_{source}") + raw = [ + (name, column_type.replace(",", ", ")) + for name, column_type in physical_columns(archetype, instance.platform) + ] + key = raw[0][0] + loaded = "'2026-03-0{day} 10:00:00'::TIMESTAMP_NTZ" + if archetype.append: + columns = [ + *raw, + ("_seq", "NUMBER(38, 0)"), + ("_loaded_at", "TIMESTAMP_NTZ"), + ("_meta", "VARIANT"), + ] + rows = [ + {key: "'K1'", "_seq": "1", "_loaded_at": loaded.format(day=1)}, + {key: "'K1'", "_seq": "2", "_loaded_at": loaded.format(day=2)}, + {key: "'K2'", "_seq": "3", "_loaded_at": loaded.format(day=2)}, + ] + expected = [("'K1'", "2"), ("'K2'", "3")] + behaviour = "keeps_the_last_delivery_of_each_row" + else: + columns = [ + *raw, + ("_op", "VARCHAR"), + ("_seq", "NUMBER(38, 0)"), + ("_loaded_at", "TIMESTAMP_NTZ"), + ("_meta", "VARIANT"), + ] + rows = [ + {key: "'K1'", "_op": "'I'", "_seq": "1", "_loaded_at": loaded.format(day=1)}, + {key: "'K1'", "_op": "'U'", "_seq": "2", "_loaded_at": loaded.format(day=2)}, + {key: "'K2'", "_op": "'I'", "_seq": "3", "_loaded_at": loaded.format(day=1)}, + {key: "'K2'", "_op": "'D'", "_seq": "4", "_loaded_at": loaded.format(day=3)}, + ] + expected = [("'K1'", "2")] + behaviour = "keeps_the_latest_live_version" + expected_rows = _expected([key, "_seq"], [[value, seq] for value, seq in expected]) + _write_test( + project, + model, + behaviour, + [ + (f"__source__{source}", _rows(columns, rows)), + (f"__expected__{model.name}", expected_rows), + ], + ) + + +def conform_test(project: Project, staged: StagedTable, conformed: Model) -> None: + """Statuses mapped to the shared vocabulary and minor units converted by currency exponent.""" + instance, archetype = staged.instance, staged.archetype + platform = instance.platform + stg = project.ref(staged.stg) + key = archetype.key + currency = CURRENCY_EXPONENT[instance.currency] + if platform == "classic": + rows = [ + {key: "'K1'", "status_code": "10", "gross_amount": "12.00"}, + {key: "'K2'", "status_code": "90", "gross_amount": "7.00"}, + ] + expected = [("'K1'", "'PENDING'", "12.00"), ("'K2'", "'CANCELLED'", "7.00")] + else: + minor = 10**currency + rows = [ + { + key: "'K1'", + "status": "'shipped'", + "gross_amount_minor": str(12 * minor), + "currency_code": f"'{instance.currency}'", + }, + { + key: "'K2'", + "status": "'delivered'", + "gross_amount_minor": str(7 * minor), + "currency_code": f"'{instance.currency}'", + }, + ] + expected = [("'K1'", "'SHIPPED'", "12.00"), ("'K2'", "'DELIVERED'", "7.00")] + fixtures = [ + (f"__ref__{stg.name}", _rows(_model_columns(stg), rows)), + _instances_fixture(staged), + ] + if '__seed("currencies")' in conformed.body: + fixtures.append( + ( + "__seed__currencies", + f" SELECT\n '{instance.currency}' AS currency_code,\n" + f" {currency} AS minor_unit_exponent", + ) + ) + expected_rows = _expected( + [key, "status", "gross_amount"], + [[k, st, f"{a}::NUMBER(18, 2)"] for k, st, a in expected], + ) + fixtures.append((f"__expected__{conformed.name}", expected_rows)) + _write_test(project, conformed, "maps_statuses_and_converts_amounts", fixtures) + + +def sequence_test(project: Project, model: Model, source: Model) -> None: + rows = [ + { + "order_id": "'O1'", + "customer_id": "'C1'", + "placed_at_utc": "'2026-03-01 09:00:00'::TIMESTAMP_NTZ", + "gross_amount": "10.00", + }, + { + "order_id": "'O2'", + "customer_id": "'C1'", + "placed_at_utc": "'2026-03-04 09:00:00'::TIMESTAMP_NTZ", + "gross_amount": "5.00", + }, + { + "order_id": "'O3'", + "customer_id": "CAST(NULL AS VARCHAR)", + "placed_at_utc": "'2026-03-02 09:00:00'::TIMESTAMP_NTZ", + "gross_amount": "1.00", + }, + ] + expected = _expected( + [ + "order_id", + "customer_sequence_number", + "days_since_previous", + "is_first", + "running_gross_amount", + ], + [ + ["'O1'", "1", "CAST(NULL AS INT)", "TRUE", "10.00::NUMBER(18, 2)"], + ["'O2'", "2", "3", "FALSE", "15.00::NUMBER(18, 2)"], + ], + ) + _write_test( + project, + model, + "numbers_orders_per_customer_and_skips_guests", + [ + (f"__ref__{source.name}", _rows(_model_columns(source), rows)), + (f"__expected__{model.name}", expected), + ], + ) + + +def enrich_test( + project: Project, model: Model, header: Model, children: list[Model], key: str +) -> None: + fixtures = [ + (f"__ref__{header.name}", _rows(_model_columns(header), [{key: "'H1'"}, {key: "'H2'"}])) + ] + for child in children: + child_key = next(c.name for c in child.columns if "unique" in c.audits) + fixtures.append( + ( + f"__ref__{child.name}", + _rows( + _model_columns(child), + [{child_key: "'L1'", key: "'H1'"}, {child_key: "'L2'", key: "'H1'"}], + ), + ) + ) + counts = [f"{child.name.split('__', 1)[1]}_count" for child in children] + expected = _expected( + [key, *counts], [["'H1'", *("2" for _ in counts)], ["'H2'", *("0" for _ in counts)]] + ) + fixtures.append((f"__expected__{model.name}", expected)) + _write_test(project, model, "counts_children_and_keeps_headers_without_any", fixtures) + + +def child_flatten_test(project: Project, child: Model, parent: Model, key: str, array: str) -> None: + rows = [ + {key: "'P1'", array: "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT(), OBJECT_CONSTRUCT())"}, + {key: "'P2'", array: "ARRAY_CONSTRUCT()"}, + ] + child_key = next(c.name for c in child.columns if "unique" in c.audits) + expected = _expected([child_key, "element_index"], [["'P1-0'", "0"], ["'P1-1'", "1"]]) + _write_test( + project, + child, + "emits_one_row_per_array_element", + [ + (f"__ref__{parent.name}", _rows(_model_columns(parent), rows)), + (f"__expected__{child.name}", expected), + ], + ) + + +MACRO_TESTS: dict[str, tuple[str | None, str]] = { + "tests/unit/macros/sql/test_money_from_minor__converts_by_currency_exponent.sql": ( + "macro", + """WITH cases AS ( + SELECT + v.case_name, + v.amount_minor, + v.exponent + FROM VALUES + ('two_decimals', 1250, 2), + ('three_decimals_rounded', 12345, 3), + ('zero_decimals', 1000, 0), + ('null_amount', NULL, 2) AS v(case_name, amount_minor, exponent) +), + +__macro_actual__ AS ( + SELECT + c.case_name, + @sql.money_from_minor(c.amount_minor, c.exponent) AS amount + FROM cases AS c +), + +__macro_expected__ AS ( + SELECT + v.case_name, + v.amount::NUMBER(38, 6) AS amount + FROM VALUES + ('two_decimals', 12.5), + ('three_decimals_rounded', 12.35), + ('zero_decimals', 1000), + ('null_amount', NULL) AS v(case_name, amount) +) +""", + ), + "tests/unit/macros/sql/test_percent_of__returns_null_for_a_zero_whole.sql": ( + "macro", + """WITH cases AS ( + SELECT + v.case_name, + v.part, + v.whole + FROM VALUES + ('quarter', 25, 100), + ('third_rounded', 1, 3), + ('zero_whole', 5, 0) AS v(case_name, part, whole) +), + +__macro_actual__ AS ( + SELECT + c.case_name, + @sql.percent_of(c.part, c.whole) AS share_percent + FROM cases AS c +), + +__macro_expected__ AS ( + SELECT + v.case_name, + v.share_percent::NUMBER(9, 4) AS share_percent + FROM VALUES + ('quarter', 25), + ('third_rounded', 33.33), + ('zero_whole', NULL) AS v(case_name, share_percent) +) +""", + ), + "tests/unit/functions/sql/test_udf__sku_family__extracts_leading_letters.sql": ( + None, + """WITH input_values AS ( + SELECT + v.sku + FROM VALUES + ('DUCK-MAL-01'), + ('decoy-teal-1'), + ('123-POND'), + (NULL) AS v(sku) +), + +__udf_actual__ AS ( + SELECT + i.sku, + __udf("udf__sku_family")(i.sku) AS sku_family + FROM input_values AS i +), + +__udf_expected__ AS ( + SELECT + v.sku::VARCHAR AS sku, + v.sku_family::VARCHAR AS sku_family + FROM VALUES + ('DUCK-MAL-01', 'DUCK'), + ('decoy-teal-1', 'DECOY'), + ('123-POND', NULL), + (NULL, NULL) AS v(sku, sku_family) +) +""", + ), + "tests/unit/functions/python/test_luhn_valid__checks_card_number_checksums.sql": ( + "udf", + """WITH input_values AS ( + SELECT + v.digits + FROM VALUES + ('79927398713'), + ('79927398710'), + ('4629173058461920'), + ('4000-0566'), + ('') AS v(digits) +), + +__udf_actual__ AS ( + SELECT + i.digits, + __udf("luhn_valid")(i.digits) AS is_valid + FROM input_values AS i +), + +__udf_expected__ AS ( + SELECT + v.digits::VARCHAR AS digits, + v.is_valid::BOOLEAN AS is_valid + FROM VALUES + ('79927398713', TRUE), + ('79927398710', FALSE), + ('4629173058461920', TRUE), + ('4000-0566', NULL), + ('', NULL) AS v(digits, is_valid) +) +""", + ), + "tests/unit/functions/sql/test_table_fn__customer_orders" + "__returns_nothing_for_unknown_customers.sql": ( + "table_fn", + """WITH __table_fn_actual__ AS ( + SELECT + o.order_id, + o.status + FROM __table_fn("table_fn__customer_orders")('DS-NO-SUCH-CUSTOMER') AS o +), + +__table_fn_expected__ AS ( + SELECT + CAST(NULL AS VARCHAR) AS order_id, + CAST(NULL AS VARCHAR) AS status + WHERE FALSE +) +""", + ), +} + + +def build_tests( + project: Project, tables: dict[str, dict[str, StagedTable]], result: Intermediate +) -> None: + staged = [table for code in sorted(tables) for _name, table in sorted(tables[code].items())] + for table in staged[::SAMPLE_STRIDE]: + base_dedupe_test(project, table) + for code in sorted(tables): + models = result.per_instance.get(code, {}) + shop = tables[code].get("orders") + if shop is None or shop.instance.system != "storefront": + continue + conform_test(project, shop, models["orders"]) + sequence_test(project, models["orders_by_customer"], models["orders"]) + enrich_test( + project, + models["orders_enriched"], + models["orders"], + [models["order_lines"], models["payments"], models["promotion_redemptions"]], + "order_id", + ) + lines = tables[code]["order_lines"] + child_flatten_test( + project, + project.ref(lines.children["discounts"]), + project.ref(lines.stg), + "order_line_id", + "discounts", + ) + for path, (mode, body) in MACRO_TESTS.items(): + if "table_fn__customer_orders" in path and "fct_orders" not in project.models: + continue + name = path.rsplit("/", 1)[1].removeprefix("test_").removesuffix(".sql") + options = f'mode {mode}, name "{name}"' if mode else f'name "{name}"' + project.write(path, f"TEST ({options});\n\n{body}") + + +def _source_fixture(staged: StagedTable, rows: list[dict[str, str]]) -> tuple[str, str]: + """A source fixture; row keys use logical stems and are mapped to the platform's names.""" + instance, archetype = staged.instance, staged.archetype + names = {archetype.key: physical_name(Field(archetype.key, "key"), instance.platform)} + names.update( + { + item.stem: physical_name(item, instance.platform) + for item in archetype.fields + if item.role in ("id", "sku", "qty", "int", "currency") + } + ) + rows = [{names.get(column, column): value for column, value in row.items()} for row in rows] + raw = [ + (name, column_type.replace(",", ", ")) + for name, column_type in physical_columns(archetype, instance.platform) + ] + meta = [("_seq", "NUMBER(38, 0)"), ("_loaded_at", "TIMESTAMP_NTZ"), ("_meta", "VARIANT")] + if not archetype.append: + meta.insert(0, ("_op", "VARCHAR")) + return f"__source__{instance.code}__{archetype.table}", _rows([*raw, *meta], rows) + + +def _scenario( + project: Project, path: str, description: str, tags: list[str], fixtures: list[tuple[str, str]] +) -> None: + blocks = [f"{name} AS (\n{body}\n)" for name, body in fixtures] + tag_list = ", ".join(f'"{tag}"' for tag in tags) + header = f'SCENARIO (\n description "{description}",\n tags [{tag_list}],\n);\n\n' + project.write(path, header + "WITH " + ",\n\n".join(blocks) + "\n") + + +def build_scenarios(project: Project, tables: dict[str, dict[str, StagedTable]]) -> None: + shop = next( + ( + tables[code] + for code in sorted(tables) + if code.startswith("shop_") and tables[code]["orders"].instance.platform == "nest" + ), + None, + ) + if shop is not None: + instance = shop["orders"].instance + loaded = "'2026-03-02 06:00:00'::TIMESTAMP_NTZ" + placed = "'2026-03-01 18:30:00 +01:00'::TIMESTAMP_TZ" + live = {"_op": "'I'", "_loaded_at": loaded} + currency = f"'{instance.currency}'" + fixtures = [ + _source_fixture( + shop["orders"], + [ + { + "order_id": "'O1'", + "customer_id": "'C1'", + "currency_code": currency, + "placed_at": placed, + "status": "'paid'", + **live, + "_seq": "1", + }, + { + "order_id": "'O2'", + "customer_id": "'C2'", + "currency_code": currency, + "placed_at": placed, + "status": "'paid'", + **live, + "_seq": "2", + }, + {"order_id": "'O2'", "_op": "'D'", "_seq": "3", "_loaded_at": loaded}, + ], + ), + _source_fixture( + shop["order_lines"], + [ + { + "order_line_id": "'L1'", + "order_id": "'O1'", + "sku": "'duck-mal-01'", + "quantity": "2", + **live, + "_seq": "1", + }, + { + "order_line_id": "'L2'", + "order_id": "'O1'", + "sku": "'duck-teal-02'", + "quantity": "1", + **live, + "_seq": "2", + }, + ], + ), + _source_fixture( + shop["payments"], + [ + { + "payment_id": "'P1'", + "order_id": "'O1'", + "currency_code": currency, + **live, + "_seq": "1", + }, + ], + ), + _source_fixture( + shop["promotion_redemptions"], + [{"_seq": "1", **live, "promotion_redemption_id": "'R1'", "order_id": "'O9'"}], + ), + ( + "__seed__currencies", + f" SELECT\n {currency} AS currency_code,\n" + f" {CURRENCY_EXPONENT[instance.currency]} AS minor_unit_exponent", + ), + _instances_fixture(shop["orders"]), + ( + f"__expected__int_{instance.code}__orders_enriched", + _expected( + [ + "order_id", + "status", + "order_lines_count", + "payments_count", + "promotion_redemptions_count", + ], + [["'O1'", "'PAID'", "2", "1", "0"]], + ), + ), + ] + _scenario( + project, + f"tests/scenarios/storefronts/{instance.code}_orders__counts_live_lines_and_payments.sql", + "Raw change logs of one storefront flow through dedupe, typing and conforming into " + "enriched orders, where deleted orders drop out", + ["storefronts", "orders"], + fixtures, + ) + site = next((tables[code] for code in sorted(tables) if code.startswith("wh_")), None) + if site is not None: + instance = site["stock_movements"].instance + loaded = "'2026-03-02 06:00:00'::TIMESTAMP_NTZ" + local = ( + "'2026-03-01 0{hour}:00:00'::TIMESTAMP_NTZ" + if instance.platform == "classic" + else "'2026-03-01 0{hour}:00:00 +00:00'::TIMESTAMP_TZ" + ) + moment = {"classic": "{stem}_ts", "nest": "{stem}_at"}[instance.platform] + live = {"_op": "'I'", "_loaded_at": loaded} + fixtures = [ + _source_fixture( + site["receipts"], + [ + { + "receipt_id": "'RC1'", + "sku": "'duck-mal-01'", + "quantity": "10", + moment.format(stem="received"): local.format(hour=1), + **live, + "_seq": "1", + } + ], + ), + _source_fixture( + site["picks"], + [ + { + "pick_id": "'PK1'", + "sku": "'duck-mal-01'", + "quantity": "3", + moment.format(stem="picked"): local.format(hour=3), + **live, + "_seq": "1", + } + ], + ), + _source_fixture( + site["adjustments"], + [ + { + "adjustment_id": "'AD1'", + "sku": "'duck-mal-01'", + "qty_delta": "-1", + moment.format(stem="adjusted"): local.format(hour=4), + **live, + "_seq": "1", + } + ], + ), + _instances_fixture(site["receipts"]), + _source_fixture( + site["stock_movements"], + [ + { + "movement_id": "'MV1'", + "sku": "'duck-mal-01'", + "qty_delta": "5", + moment.format(stem="moved"): local.format(hour=2), + "_seq": "1", + "_loaded_at": loaded, + } + ], + ), + ( + f"__expected__int_{instance.code}__stock_ledger", + _expected( + ["movement_key", "qty_delta", "on_hand_after"], + [ + ["'receipt:RC1'", "10", "10"], + ["'movement:MV1'", "5", "15"], + ["'pick:PK1'", "-3", "12"], + ["'adjustment:AD1'", "-1", "11"], + ], + ), + ), + ] + _scenario( + project, + f"tests/scenarios/warehouses/{instance.code}_stock__runs_the_ledger_in_time_order.sql", + "Receipts, movements, picks and adjustments of one warehouse become a signed ledger " + "with the running quantity on hand", + ["warehouses", "inventory"], + fixtures, + ) diff --git a/benchmarks/duckshop-10k/duckshop10k/world.py b/benchmarks/duckshop-10k/duckshop10k/world.py new file mode 100644 index 0000000000..3c26b03bcb --- /dev/null +++ b/benchmarks/duckshop-10k/duckshop10k/world.py @@ -0,0 +1,508 @@ +"""The fictional business DuckShop 10K describes: Mallard & Co. after a decade of growth. + +Everything here is invented. Instance codes are built from country codes, city names, product +categories and generic channel words, so no code names a real company, brand or product. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Region: + code: str + name: str + timezone: str + + +@dataclass(frozen=True) +class Storefront: + country: str + region: str + timezone: str + currency: str + platform: str + + @property + def code(self) -> str: + return f"shop_{self.country.lower()}" + + +@dataclass(frozen=True) +class Instance: + """One deployment of a source system: a marketplace, a warehouse, a carrier and so on.""" + + system: str + code: str + region: str + timezone: str + currency: str + platform: str + label: str + + +REGIONS: tuple[Region, ...] = ( + Region("eu_west", "Western Europe", "Europe/Berlin"), + Region("eu_south", "Southern Europe", "Europe/Madrid"), + Region("eu_north", "Northern Europe", "Europe/Stockholm"), + Region("eu_east", "Central and Eastern Europe", "Europe/Warsaw"), + Region("uk_ie", "United Kingdom and Ireland", "Europe/London"), + Region("north_america", "North America", "America/New_York"), + Region("latam", "Latin America", "America/Sao_Paulo"), + Region("asia", "Asia", "Asia/Tokyo"), + Region("oceania", "Oceania", "Australia/Sydney"), + Region("mea", "Middle East and Africa", "Asia/Dubai"), +) +REGION_BY_CODE: dict[str, Region] = {region.code: region for region in REGIONS} + +CURRENCY_EXPONENT: dict[str, int] = { + "AED": 2, + "AUD": 2, + "BHD": 3, + "BRL": 2, + "CAD": 2, + "CHF": 2, + "CLP": 0, + "COP": 2, + "CZK": 2, + "DKK": 2, + "EUR": 2, + "GBP": 2, + "HUF": 2, + "INR": 2, + "JPY": 0, + "KRW": 0, + "KWD": 3, + "MXN": 2, + "NOK": 2, + "NZD": 2, + "PLN": 2, + "QAR": 2, + "SAR": 2, + "SEK": 2, + "SGD": 2, + "THB": 2, + "TWD": 2, + "USD": 2, + "ZAR": 2, +} + +# Platform generations of the storefront software. `classic` is the original monolith with +# legacy column names, local timestamps and JSON kept as text; `nest` is the columnar rewrite; +# `flock` is the document store that newer storefronts launched on. +PLATFORMS: tuple[str, ...] = ("classic", "nest", "flock") + +_STOREFRONT_ROWS: tuple[tuple[str, str, str, str, str], ...] = ( + ("DE", "eu_west", "Europe/Berlin", "EUR", "classic"), + ("FR", "eu_west", "Europe/Paris", "EUR", "classic"), + ("NL", "eu_west", "Europe/Amsterdam", "EUR", "classic"), + ("AT", "eu_west", "Europe/Vienna", "EUR", "nest"), + ("CH", "eu_west", "Europe/Zurich", "CHF", "nest"), + ("BE", "eu_west", "Europe/Brussels", "EUR", "nest"), + ("LU", "eu_west", "Europe/Luxembourg", "EUR", "flock"), + ("ES", "eu_south", "Europe/Madrid", "EUR", "nest"), + ("PT", "eu_south", "Europe/Lisbon", "EUR", "nest"), + ("IT", "eu_south", "Europe/Rome", "EUR", "nest"), + ("GR", "eu_south", "Europe/Athens", "EUR", "flock"), + ("SE", "eu_north", "Europe/Stockholm", "SEK", "nest"), + ("NO", "eu_north", "Europe/Oslo", "NOK", "nest"), + ("DK", "eu_north", "Europe/Copenhagen", "DKK", "nest"), + ("FI", "eu_north", "Europe/Helsinki", "EUR", "flock"), + ("PL", "eu_east", "Europe/Warsaw", "PLN", "nest"), + ("CZ", "eu_east", "Europe/Prague", "CZK", "flock"), + ("HU", "eu_east", "Europe/Budapest", "HUF", "flock"), + ("GB", "uk_ie", "Europe/London", "GBP", "classic"), + ("IE", "uk_ie", "Europe/Dublin", "EUR", "nest"), + ("US", "north_america", "America/New_York", "USD", "classic"), + ("CA", "north_america", "America/Toronto", "CAD", "nest"), + ("MX", "north_america", "America/Mexico_City", "MXN", "flock"), + ("BR", "latam", "America/Sao_Paulo", "BRL", "flock"), + ("CL", "latam", "America/Santiago", "CLP", "flock"), + ("CO", "latam", "America/Bogota", "COP", "flock"), + ("JP", "asia", "Asia/Tokyo", "JPY", "classic"), + ("KR", "asia", "Asia/Seoul", "KRW", "nest"), + ("SG", "asia", "Asia/Singapore", "SGD", "nest"), + ("IN", "asia", "Asia/Kolkata", "INR", "flock"), + ("TW", "asia", "Asia/Taipei", "TWD", "flock"), + ("TH", "asia", "Asia/Bangkok", "THB", "flock"), + ("AU", "oceania", "Australia/Sydney", "AUD", "classic"), + ("NZ", "oceania", "Pacific/Auckland", "NZD", "nest"), + ("AE", "mea", "Asia/Dubai", "AED", "nest"), + ("SA", "mea", "Asia/Riyadh", "SAR", "flock"), + ("KW", "mea", "Asia/Kuwait", "KWD", "classic"), + ("BH", "mea", "Asia/Bahrain", "BHD", "flock"), + ("QA", "mea", "Asia/Qatar", "QAR", "flock"), + ("ZA", "mea", "Africa/Johannesburg", "ZAR", "flock"), +) + +STOREFRONTS: tuple[Storefront, ...] = tuple(Storefront(*row) for row in _STOREFRONT_ROWS) + +# Duck species name the product lines, as in the base DuckShop project. +PRODUCT_LINES: tuple[str, ...] = ( + "mallard", + "pintail", + "teal", + "eider", + "merganser", + "wigeon", + "shoveler", + "gadwall", + "garganey", + "goldeneye", + "scaup", + "pochard", +) + +TEAMS: tuple[str, ...] = ("finance", "marketing", "operations", "support", "merchandising") + + +def _storefront_for(country: str) -> Storefront: + return next(storefront for storefront in STOREFRONTS if storefront.country == country) + + +def _instance(system: str, code: str, country: str, platform: str, label: str) -> Instance: + storefront = _storefront_for(country) + return Instance( + system=system, + code=code, + region=storefront.region, + timezone=storefront.timezone, + currency=storefront.currency, + platform=platform, + label=label, + ) + + +def _cycle(values: tuple[str, ...], index: int) -> str: + return values[index % len(values)] + + +_MARKETPLACE_CATEGORIES: tuple[str, ...] = ("bazaar", "crafts", "home", "gifts", "toys", "garden") +_MARKETPLACE_COUNTRIES: tuple[str, ...] = ("DE", "GB", "US", "JP", "FR", "BR", "AU", "AE") + +_WAREHOUSE_SITES: tuple[tuple[str, str, str], ...] = ( + ("pond", "leipzig", "DE"), + ("nest", "columbus", "US"), + ("marsh", "osaka", "JP"), + ("reed", "venlo", "NL"), + ("brook", "lyon", "FR"), + ("delta", "zaragoza", "ES"), + ("lagoon", "verona", "IT"), + ("fen", "goteborg", "SE"), + ("mere", "poznan", "PL"), + ("tarn", "coventry", "GB"), + ("creek", "reno", "US"), + ("bayou", "dallas", "US"), + ("estuary", "toronto", "CA"), + ("shoal", "monterrey", "MX"), + ("spring", "campinas", "BR"), + ("rill", "santiago", "CL"), + ("weir", "incheon", "KR"), + ("cove", "jurong", "SG"), + ("beck", "pune", "IN"), + ("loch", "dublin", "IE"), + ("haven", "melbourne", "AU"), + ("sound", "auckland", "NZ"), + ("oasis", "jebel_ali", "AE"), + ("wadi", "dammam", "SA"), + ("vlei", "midrand", "ZA"), + ("kanal", "brno", "CZ"), +) + +_STORE_CITIES: tuple[tuple[str, str], ...] = ( + ("berlin", "DE"), + ("hamburg", "DE"), + ("munich", "DE"), + ("cologne", "DE"), + ("paris", "FR"), + ("bordeaux", "FR"), + ("amsterdam", "NL"), + ("vienna", "AT"), + ("zurich", "CH"), + ("madrid", "ES"), + ("barcelona", "ES"), + ("lisbon", "PT"), + ("milan", "IT"), + ("rome", "IT"), + ("stockholm", "SE"), + ("oslo", "NO"), + ("copenhagen", "DK"), + ("warsaw", "PL"), + ("london", "GB"), + ("manchester", "GB"), + ("edinburgh", "GB"), + ("bristol", "GB"), + ("dublin", "IE"), + ("new_york", "US"), + ("boston", "US"), + ("chicago", "US"), + ("seattle", "US"), + ("austin", "US"), + ("toronto", "CA"), + ("vancouver", "CA"), + ("tokyo", "JP"), + ("kyoto", "JP"), + ("seoul", "KR"), + ("singapore", "SG"), + ("sydney", "AU"), + ("melbourne", "AU"), + ("auckland", "NZ"), + ("dubai", "AE"), + ("sao_paulo", "BR"), + ("cape_town", "ZA"), +) + +_SUPPLIER_CATEGORIES: tuple[tuple[str, str], ...] = ( + ("rubber", "TH"), + ("textile", "PT"), + ("decoy", "CZ"), + ("garden", "NL"), + ("bath", "IT"), + ("ceramics", "ES"), + ("packaging", "PL"), + ("print", "DE"), +) + +_GATEWAYS: tuple[tuple[str, str], ...] = ( + ("pay_card_eu", "DE"), + ("pay_card_us", "US"), + ("pay_card_apac", "SG"), + ("pay_wallet", "GB"), + ("pay_instalments", "SE"), + ("pay_bank_transfer", "NL"), +) + +_AD_NETWORKS: tuple[tuple[str, str], ...] = ( + ("ads_search_global", "US"), + ("ads_search_asia", "JP"), + ("ads_social_global", "US"), + ("ads_social_latam", "BR"), + ("ads_video_global", "US"), + ("ads_display_global", "GB"), + ("ads_display_eu", "DE"), + ("ads_shopping_global", "US"), + ("ads_retail_media_us", "US"), + ("ads_retail_media_eu", "FR"), + ("ads_podcast_global", "GB"), + ("ads_outdoor_eu", "NL"), + ("ads_creator_global", "US"), + ("ads_price_comparison_eu", "DE"), +) + +_MESSAGING: tuple[tuple[str, str], ...] = ( + ("msg_email_lifecycle", "GB"), + ("msg_email_bulk", "DE"), + ("msg_sms", "US"), + ("msg_push", "SG"), +) + +_SUPPORT_TOOLS: tuple[tuple[str, str], ...] = ( + ("support_desk", "GB"), + ("support_chat", "DE"), + ("support_voice", "US"), +) + +_LEGAL_ENTITY_COUNTRIES: tuple[str, ...] = ( + "GB", + "DE", + "FR", + "NL", + "ES", + "IT", + "SE", + "PL", + "US", + "CA", + "JP", + "AU", + "AE", + "BR", +) + +_COLLECTORS: tuple[str, ...] = tuple(f"web_{region.code}" for region in REGIONS) + ( + "app_phone", + "app_tablet", +) +_COLLECTOR_HOME: dict[str, str] = { + "web_eu_west": "DE", + "web_eu_south": "ES", + "web_eu_north": "SE", + "web_eu_east": "PL", + "web_uk_ie": "GB", + "web_north_america": "US", + "web_latam": "BR", + "web_asia": "JP", + "web_oceania": "AU", + "web_mea": "AE", + "app_phone": "GB", + "app_tablet": "GB", +} + + +def _region_home(region: str) -> str: + return next(storefront.country for storefront in STOREFRONTS if storefront.region == region) + + +def instances() -> dict[str, tuple[Instance, ...]]: + """Every deployed source system, grouped by system kind, in a fixed order.""" + marketplaces = tuple( + _instance( + "marketplace", + f"mkt_{_cycle(_MARKETPLACE_CATEGORIES, index)}_{country.lower()}", + country, + _cycle(("nest", "flock"), index), + f"{_cycle(_MARKETPLACE_CATEGORIES, index)} marketplace partner in {country}", + ) + for index, country in enumerate( + _MARKETPLACE_COUNTRIES + _MARKETPLACE_COUNTRIES[:4] + ("IT", "ES", "MX", "KR") + ) + ) + # Make marketplace codes unique when a category repeats in one country. + seen: dict[str, int] = {} + unique_marketplaces: list[Instance] = [] + for marketplace in marketplaces: + count = seen.get(marketplace.code, 0) + seen[marketplace.code] = count + 1 + if count: + marketplace = Instance( + **{**marketplace.__dict__, "code": f"{marketplace.code}_{count}"} + ) + unique_marketplaces.append(marketplace) + carriers = ( + _instance("carrier", "carrier_flapwing", "GB", "nest", "Flapwing Parcels"), + _instance("carrier", "carrier_wingbeat", "DE", "nest", "Wingbeat Couriers"), + _instance("carrier", "carrier_webfoot", "US", "classic", "Webfoot Freight"), + *( + _instance("carrier", f"carrier_postal_{country.lower()}", country, "classic", "post") + for country in ("GB", "DE", "FR", "US", "JP", "AU", "BR", "AE") + ), + *( + _instance( + "carrier", + f"carrier_courier_{region.code}", + _region_home(region.code), + "flock", + f"regional courier for {region.name}", + ) + for region in REGIONS[3:10] + ), + ) + warehouses = tuple( + _instance( + "warehouse", + f"wh_{name}_{city}", + country, + _cycle(("classic", "nest"), index), + f"The {name.title()} fulfilment centre", + ) + for index, (name, city, country) in enumerate(_WAREHOUSE_SITES) + ) + stores = tuple( + _instance("store", f"store_{city}", country, "nest", f"retail store in {city}") + for city, country in _STORE_CITIES + ) + suppliers = tuple( + _instance( + "supplier", + f"sup_{category}_{number:02d}", + country, + _cycle(("nest", "flock", "classic"), number), + f"{category} goods supplier {number}", + ) + for category, country in _SUPPLIER_CATEGORIES + for number in range(1, 6) + ) + gateways = tuple( + _instance("gateway", code, country, "nest", code.replace("_", " ")) + for code, country in _GATEWAYS + ) + ad_networks = tuple( + _instance("ad_network", code, country, "flock", code.replace("_", " ")) + for code, country in _AD_NETWORKS + ) + messaging = tuple( + _instance("messaging", code, country, "nest", code.replace("_", " ")) + for code, country in _MESSAGING + ) + affiliates = tuple( + _instance( + "affiliate", + f"aff_{region.code}", + _region_home(region.code), + "nest", + f"affiliate network for {region.name}", + ) + for region in REGIONS[:8] + ) + collectors = tuple( + _instance("collector", code, _COLLECTOR_HOME[code], "flock", code.replace("_", " ")) + for code in _COLLECTORS + ) + support_tools = tuple( + _instance("support", code, country, "nest", code.replace("_", " ")) + for code, country in _SUPPORT_TOOLS + ) + legal_entities = tuple( + _instance( + "ledger", + f"le_{country.lower()}", + country, + "classic" if country in ("GB", "DE", "US") else "nest", + f"Mallard & Co. legal entity in {country}", + ) + for country in _LEGAL_ENTITY_COUNTRIES + ) + wholesale = tuple( + _instance( + "wholesale", + f"b2b_{region.code}", + _region_home(region.code), + "nest", + f"wholesale desk for {region.name}", + ) + for region in REGIONS + ) + returns_portals = tuple( + _instance( + "returns", + f"returns_{region.code}", + _region_home(region.code), + "flock", + f"returns portal for {region.name}", + ) + for region in REGIONS + ) + catalogue = (_instance("catalogue", "pim", "GB", "nest", "product information system"),) + return { + "marketplace": tuple(unique_marketplaces), + "carrier": carriers, + "warehouse": warehouses, + "store": stores, + "supplier": suppliers, + "gateway": gateways, + "ad_network": ad_networks, + "messaging": messaging, + "affiliate": affiliates, + "collector": collectors, + "support": support_tools, + "ledger": legal_entities, + "wholesale": wholesale, + "returns": returns_portals, + "catalogue": catalogue, + } + + +def storefront_instances() -> tuple[Instance, ...]: + return tuple( + Instance( + system="storefront", + code=storefront.code, + region=storefront.region, + timezone=storefront.timezone, + currency=storefront.currency, + platform=storefront.platform, + label=f"storefront for {storefront.country}", + ) + for storefront in STOREFRONTS + ) diff --git a/benchmarks/duckshop-10k/generate.py b/benchmarks/duckshop-10k/generate.py new file mode 100644 index 0000000000..91bdd8efc8 --- /dev/null +++ b/benchmarks/duckshop-10k/generate.py @@ -0,0 +1,65 @@ +"""Generate the DuckShop 10K project, or check a generated copy against checksum.txt. + +python benchmarks/duckshop-10k/generate.py --out /path/to/duckshop-10k +python benchmarks/duckshop-10k/generate.py --check +python benchmarks/duckshop-10k/generate.py --verify /path/to/duckshop-10k +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from duckshop10k.manifest import checksum_line, read_project, write_project # noqa: E402 +from duckshop10k.project import PROFILES, build # noqa: E402 + +CHECKSUM_FILE = HERE / "checksum.txt" + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--out", type=Path, help="write the project into this new directory") + action.add_argument( + "--check", action="store_true", help="generate in memory and compare with checksum.txt" + ) + action.add_argument( + "--verify", type=Path, help="compare a generated directory with checksum.txt" + ) + action.add_argument( + "--update-checksum", action="store_true", help="rewrite checksum.txt from a fresh build" + ) + parser.add_argument("--profile", choices=sorted(PROFILES), default="full") + arguments = parser.parse_args(argv) + + if arguments.verify is not None: + return _compare(checksum_line(read_project(arguments.verify))) + files = build(arguments.profile) + if arguments.out is not None: + write_project(files, arguments.out) + print(f"wrote {len(files)} files to {arguments.out}: {checksum_line(files).strip()}") + return 0 + if arguments.profile != "full": + parser.error("checksum.txt records the full profile only") + if arguments.update_checksum: + CHECKSUM_FILE.write_text(checksum_line(files)) + print(checksum_line(files).strip()) + return 0 + return _compare(checksum_line(files)) + + +def _compare(actual: str) -> int: + expected = CHECKSUM_FILE.read_text() + if actual != expected: + print(f"checksum mismatch\n expected: {expected.strip()}\n actual: {actual.strip()}") + return 1 + print(f"checksum matches: {actual.strip()}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/benchmarks/duckshop-10k/tests/test_generator.py b/benchmarks/duckshop-10k/tests/test_generator.py new file mode 100644 index 0000000000..5aa1fa478d --- /dev/null +++ b/benchmarks/duckshop-10k/tests/test_generator.py @@ -0,0 +1,160 @@ +"""Offline checks of the DuckShop 10K generator: shape, determinism and the recorded checksum.""" + +from __future__ import annotations + +import re +import sys +from functools import cache +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(HERE)) + +from duckshop10k.core import _placement, declaration_uses # noqa: E402 +from duckshop10k.manifest import checksum, checksum_line, read_project, write_project # noqa: E402 +from duckshop10k.project import build # noqa: E402 +from duckshop10k.stats import shape # noqa: E402 + + +@cache +def full() -> dict[str, str]: + return build("full") + + +def models(files: dict[str, str]) -> dict[str, str]: + return { + path: text + for path, text in files.items() + if path.startswith("models/") and "/_sqlbuild/" not in path and path.endswith(".sql") + } + + +def test_given_two_builds_when_fingerprinting_then_checksums_match() -> None: + assert checksum(build("full")) == checksum(full()) + + +def test_given_full_profile_when_building_then_matches_recorded_checksum() -> None: + assert checksum_line(full()) == (HERE / "checksum.txt").read_text() + + +def test_given_full_profile_when_measuring_then_has_about_ten_thousand_models_in_real_mix() -> None: + measured = shape(full()) + + assert 9_900 <= measured.models <= 10_300 + assert 58 <= measured.layer_percent("staging") <= 62 + assert 18 <= measured.layer_percent("intermediate") <= 21 + assert 19 <= measured.layer_percent("marts") + measured.layer_percent("snapshots") <= 23 + + +def test_given_full_profile_when_measuring_then_fan_in_and_depth_match_the_profile() -> None: + measured = shape(full()) + reading = {count: n for count, n in measured.upstream_counts.items() if count > 0} + total = sum(reading.values()) + + assert sum(n for count, n in reading.items() if count <= 4) / total >= 0.95 + assert 10 <= sum(n for count, n in reading.items() if count >= 10) <= 100 + assert 8 <= measured.max_depth <= 14 + + +def test_given_full_profile_when_measuring_then_samples_a_few_hundred_unit_tests() -> None: + measured = shape(full()) + + assert 200 <= measured.unit_tests <= 600 + assert 1 <= measured.scenarios <= 5 + + +@pytest.mark.parametrize( + ("construct", "minimum_models"), + [ + (r"\bHASH\(", 1), + (r"TIMESTAMP_LTZ", 1), + (r"CURRENT_TIMESTAMP\(\)", 2), + (r"ASOF JOIN", 10), + (r"LATERAL FLATTEN", 200), + (r"\bQUALIFY\b", 100), + (r"CONVERT_TIMEZONE|@sql\.(local_business_date|local_to_utc|tz_to_utc)\(", 800), + (r"TRY_PARSE_JSON", 100), + (r"\bFILTER\(", 10), + (r"incremental_mode microbatch", 20), + (r"materialized snapshot", 5), + (r"__udf\(", 2), + (r"__table_fn\(", 1), + (r"@py\.", 3), + (r"GENERATOR\(", 1), + ], +) +def test_given_full_profile_when_scanning_then_uses_each_construct( + construct: str, minimum_models: int +) -> None: + pattern = re.compile(construct) + + assert sum(1 for text in models(full()).values() if pattern.search(text)) >= minimum_models + + +def test_given_known_divergence_constructs_when_scanning_then_models_are_tagged() -> None: + for path, text in models(full()).items(): + if re.search(r"\bHASH\(|TIMESTAMP_LTZ|CURRENT_TIMESTAMP\(\)", text.split(");", 1)[1]): + assert "known_divergence" in text, path + + +def test_given_generated_files_when_scanning_then_uses_only_synthetic_identifiers() -> None: + joined = "\n".join(full().values()).lower() + + for word in ("password", "secret", "token", "@gmail", "snowflakecomputing"): + assert word not in joined + + +def test_given_declarations_when_placing_then_follow_their_users() -> None: + assert _placement({"models/staging/a/x", "models/staging/b/y"}, "macros/sql") == ( + "models/staging/_sqlbuild/macros/sql" + ) + assert _placement({"models/staging/a"}, "constants") == "models/staging/a/_sqlbuild/_constants" + assert _placement({"models/staging/a", "tests/unit/b"}, "macros/sql") == "macros/sql" + assert _placement({"models/a", "models/b"}, "enums") == "enums" + + +def test_given_full_profile_when_placing_then_every_used_declaration_is_written_once() -> None: + files = full() + declared = [path for path in files if "/_sqlbuild/" in path or path.startswith("macros/")] + names = [Path(path).name for path in declared] + + assert len(names) == len(set(names)) + used = set().union(*(declaration_uses(text) for text in models(files).values())) + assert ("macro", "cdc_latest") in used + assert any(path.endswith("/cdc_latest.sql") for path in declared) + + +def test_given_written_project_when_reading_back_then_checksum_is_unchanged(tmp_path: Path) -> None: + files = build("smoke") + write_project(files, tmp_path / "project") + + assert checksum(read_project(tmp_path / "project")) == checksum(files) + + +def test_given_non_empty_output_when_writing_then_refuses(tmp_path: Path) -> None: + (tmp_path / "existing.txt").write_text("keep me") + + with pytest.raises(SystemExit): + write_project(build("smoke"), tmp_path) + + +def test_given_generated_custom_rule_when_running_its_tests_then_they_pass(tmp_path: Path) -> None: + project = tmp_path / "project" + write_project(build("smoke"), project) + sys.path.insert(0, str(project)) + try: + namespace: dict[str, object] = {} + exec( + compile((project / "tests/test_rule_staging_dedupe.py").read_text(), "rule", "exec"), + namespace, + ) + tests = [value for name, value in namespace.items() if name.startswith("test_")] + assert len(tests) == 3 + for test in tests: + test() # type: ignore[operator] + finally: + sys.path.remove(str(project)) + for module in [name for name in sys.modules if name.startswith("rules")]: + del sys.modules[module] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index fad2af844a..8ea2bac6dd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1249,3 +1249,14 @@ class BenchmarkProjectCompileTestCase: project_dir: Path expected_errors: int expected_warnings: int + + +@dataclass(frozen=True) +class GeneratedBenchmarkCompileTestCase: + """A generated benchmark project, its recorded checksum and its cold compile budget.""" + + description: str + generator: Path + expected_min_models: int + expected_max_wall_seconds: float + expected_max_rss_bytes: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index a2fedadab9..291e7ac39e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5324,3 +5324,17 @@ def without_compile_timings(*, report: str) -> str: """A JSON report without its compile timings, which differ between compiles.""" return _COMPILE_TIMINGS_PATTERN.sub("", report) + + +def run_benchmark_generator( + *, generator: Path, arguments: tuple[str, ...] +) -> subprocess.CompletedProcess[str]: + """Run a benchmark project generator script with this interpreter and capture its output.""" + + return subprocess.run( + (sys.executable, str(generator), *arguments), + capture_output=True, + text=True, + check=False, + timeout=300, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py new file mode 100644 index 0000000000..57804a95e7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_10k_compile.py @@ -0,0 +1,78 @@ +"""Fresh-process guard that the generated DuckShop 10K project is reproducible and compiles cleanly. + +The guard runs in the dense 10,000-model compile job, under the same 4 GiB memory limit, because a +cold compile of ten thousand models is too slow for the ordinary E2E shards. +""" + +import subprocess +from pathlib import Path +from typing import cast + +import pytest + +from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( + assert_required_cgroup_memory_limit, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + GeneratedBenchmarkCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + FreshProcessCompileBenchmarkResult, + _run_fresh_process_compile_benchmark, + run_benchmark_generator, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT + +_GIB: int = 1024 * 1024 * 1024 + + +@pytest.mark.performance +@pytest.mark.cold_compile_performance +@pytest.mark.parametrize( + "test_case", + ( + GeneratedBenchmarkCompileTestCase( + "duckshop_models_10000_generated", + REPO_ROOT / "benchmarks" / "duckshop-10k" / "generate.py", + 9_900, + 60.0, + 3 * _GIB, + ), + ), + ids=lambda case: case.description, +) +def test_given_generated_duckshop_10k_when_compiling_cold_then_matches_checksum_and_is_clean( + test_case: GeneratedBenchmarkCompileTestCase, tmp_path: Path +) -> None: + assert_required_cgroup_memory_limit() + project_dir: Path = tmp_path / "duckshop_10k" + + generated: subprocess.CompletedProcess[str] = run_benchmark_generator( + generator=test_case.generator, arguments=("--out", str(project_dir)) + ) + assert generated.returncode == 0, generated.stdout + generated.stderr + verified: subprocess.CompletedProcess[str] = run_benchmark_generator( + generator=test_case.generator, arguments=("--verify", str(project_dir)) + ) + assert verified.returncode == 0, verified.stdout + verified.stderr + + result: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="duckshop-10k", + expected_max_wall_seconds=test_case.expected_max_wall_seconds, + compile_args=("--no-cache",), + ) + summary: dict[str, int] = cast(dict[str, int], result.payload["summary"]) + print( + f"duckshop 10k compile summary={summary} wall={result.elapsed_seconds:.2f}s " + f"cpu={result.cpu_seconds:.2f}s peak_rss_mib={result.peak_rss_bytes // (1024 * 1024)}", + flush=True, + ) + assert result.payload["diagnostics"] == [] + assert result.payload["has_errors"] is False + assert summary["errors"] == 0 + assert summary["warnings"] == 0 + assert summary["models"] >= test_case.expected_min_models + assert summary["tests"] > 0 + assert result.elapsed_seconds < test_case.expected_max_wall_seconds + assert result.peak_rss_bytes < test_case.expected_max_rss_bytes