From 9146bd0d46fca7ba9a7303f242f30f7987c3ea6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:18:33 +0100 Subject: [PATCH 1/4] feat(compile)!: require a description on every named resource Missing descriptions are a P010 compile error for models, scenarios, seeds, sources, SQL and Python functions, named SQL and Python hooks, loaders, tasks, assets, checks and providers. Each finding points at the declaration and shows the exact header, YAML or decorator snippet to add. FUNCTION headers and @udf accept a new description (docstring fallback for Python UDFs), and @loader accepts description= with a docstring fallback. format --check and format now fault on missing MODEL, SCENARIO, FUNCTION and HOOK header descriptions, resolving [defaults], [path_defaults] and model_schema descriptions exactly as compile does. SQBRTEST105 is retired in favour of the compiler requirement. BREAKING CHANGE: projects with undescribed resources no longer compile. SQBRTEST105 was replaced by P010 and must be removed from [rules] select and ignore selectors, which otherwise fail with C000. --- .../src/engine/tests/test_sql_test_rules.rs | 26 - .../src/rules/_helpers/catalogue.rs | 8 - .../src/rules/_helpers/sql_test_rules.rs | 24 - .../cli/commands/_helpers/compile/output.py | 1 + .../cli/commands/_helpers/playground/copy.py | 2 + .../models/customer_revenue.sql | 2 +- .../models/fact_waffle_orders.sql | 2 +- .../python/loaders/waffle_loaders.py.txt | 6 + .../loader_waffle_shop/sources/raw.yml | 2 + .../sql/table_fn__customer_orders.sql | 1 + .../functions/sql/udf__is_completed_order.sql | 1 + .../templates/waffle_shop/sources/raw.yml | 3 + .../compile/_helpers/assembly/project.py | 9 +- .../compile/_helpers/attachment/audits.py | 27 + .../compile/_helpers/attachment/core.py | 26 +- .../compile/_helpers/attachment/functions.py | 23 + .../_helpers/attachment/model_config.py | 32 + .../_helpers/diagnostics/descriptions.py | 458 ++++++++++++ .../classes/model_description_resolution.py | 41 ++ src/sqlbuild/compiler/compile/constants.py | 33 + .../compile/main/_build_compile_inputs.py | 36 +- src/sqlbuild/compiler/compile/models.py | 2 + .../discovery/_helpers/filesystem/core.py | 1 + .../_helpers/integrations/loaders.py | 18 +- .../discovery/_helpers/python/functions.py | 5 + src/sqlbuild/compiler/discovery/constants.py | 3 +- .../main/_model_description_inputs.py | 37 + .../discovery/main/_yaml_entry_line.py | 24 + src/sqlbuild/compiler/discovery/models.py | 1 + .../_helpers/warehouse/source_tables.py | 21 +- .../python_nodes/_helpers/inventory.py | 1 + .../errors/setting_help/_helpers/text.py | 5 + .../errors/setting_help/main/snippet_help.py | 11 + src/sqlbuild/lint/_helpers/native.py | 36 +- src/sqlbuild/lint/constants.py | 4 +- src/sqlbuild/lint/main/run_format.py | 4 +- src/sqlbuild/loaders.pyi | 1 + .../python_nodes/_helpers/loader_authoring.py | 2 + .../python_nodes/main/_apply_loader.py | 3 + src/sqlbuild/python_nodes/models.py | 1 + .../rule_engine/_helpers/guidance/skills.py | 1 - .../cli/commands/main/compile/_test_types.py | 43 ++ .../cli/commands/main/compile/helpers.py | 52 ++ .../compile/test_required_descriptions.py | 669 ++++++++++++++++++ .../cli/commands/main/rules/_test_types.py | 1 + .../cli/commands/main/rules/helpers.py | 18 + .../rules/test_rules_cache_invalidation.py | 25 + .../sqlbuild/cli/commands/main/_test_types.py | 6 +- .../sqlbuild/cli/commands/main/test_format.py | 33 +- .../compiler/compile/_helpers/_test_types.py | 30 + .../config/test_model_config_reuse.py | 8 +- .../compiler/compile/_helpers/helpers.py | 282 +++++++- .../compile/_helpers/test_descriptions.py | 229 ++++++ .../discovery/_helpers/_test_types.py | 7 +- .../_helpers/test_filesystem_loaders.py | 50 ++ .../_helpers/test_function_header_keys.py | 62 +- .../discovery/_helpers/test_yml_sources.py | 4 + .../setting_help/main/test_setting_help.py | 12 + .../src/sqlbuild/lint/_helpers/test_native.py | 19 +- tests/unit/src/sqlbuild/lint/_test_types.py | 10 + tests/unit/src/sqlbuild/lint/test_run_lint.py | 63 +- .../_helpers/engine/test_catalogue.py | 2 - .../models/finance/daily_revenue.sql | 1 + .../tidy-shop/models/daily_revenue.sql | 1 + .../sql/table_fn__customer_orders.sql | 1 + .../functions/sql/udf__is_completed_order.sql | 1 + website/examples/waffle-shop/sources/raw.yml | 3 + 67 files changed, 2418 insertions(+), 158 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py create mode 100644 src/sqlbuild/compiler/compile/classes/model_description_resolution.py create mode 100644 src/sqlbuild/compiler/discovery/main/_model_description_inputs.py create mode 100644 src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py create mode 100644 src/sqlbuild/errors/setting_help/main/snippet_help.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs index 1545d99e54..a503f7d2e2 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs @@ -191,32 +191,6 @@ fn given_sql_test_facts_when_evaluating_project_rules_then_returns_expected_faul "tests/unit/test_orders__paid.sql", ], }, - test_types::SqlTestRulesTestCase { - description: "scenario descriptions reject generic numbering", - code: "SQBRTEST105", - tests: json!([]), - scenarios: json!([ - { - "source_path": "tests/scenarios/orders__paid.sql", - "ownership_root": "tests/scenarios", "name": "orders__paid", - "description": "Paid orders remain visible", "expected_model_names": [], - "assertion_names": [], "assertion_target_model_names": [], - "target_model_names": [] - }, - { - "source_path": "tests/scenarios/orders__case_1.sql", - "ownership_root": "tests/scenarios", "name": "orders__case_1", - "description": "case 1", "expected_model_names": [], - "assertion_names": [], "assertion_target_model_names": [], - "target_model_names": [] - } - ]), - scope_index: helpers::scope_index(), - config: json!({}), - expected_fault_count: 1, - expected_evaluated_models: 0, - expected_paths: &["tests/scenarios/orders__case_1.sql"], - }, test_types::SqlTestRulesTestCase { description: "path-scoped ignore suppresses a test rule finding", code: "SQBRTEST104", diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs index f5105ad6cd..5d5741fb04 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs @@ -38,7 +38,6 @@ macro_rules! rule { | "SQBRTEST102" | "SQBRTEST103" | "SQBRTEST104" - | "SQBRTEST105" | "SQBRTEST203" | "SQBRTEST301" ), @@ -294,13 +293,6 @@ pub(crate) fn catalogue() -> Vec { "every SQL unit-test block must have a target-aware subject__expected_behavior name", "Add name \"__\" to this TEST header.", ), - rule!( - "SQBRTEST105", - "tests", - "scenario-business-description", - "scenario descriptions must identify a concrete business behavior", - "Write a non-generic SCENARIO description that states the business behavior under test.", - ), rule!( "SQBRTEST201", "tests", diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs index a68c859612..6743a60084 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs @@ -90,9 +90,6 @@ pub(crate) fn evaluate_project( if let Some(rule) = evaluation.selected.get("SQBRTEST104") { faults.extend(structured_names(rule, &evaluation, &tests)); } - if let Some(rule) = evaluation.selected.get("SQBRTEST105") { - faults.extend(scenario_descriptions(rule, &scenarios)); - } if let Some(rule) = evaluation.selected.get(EMPTY_INPUT_RULE_CODE) { faults.extend(empty_input_only_faults( rule, @@ -270,27 +267,6 @@ fn structured_names( faults } -fn scenario_descriptions(rule: &RuleMetadata, scenarios: &[&SqlScenarioFact]) -> Vec { - let mut faults: Vec = Vec::new(); - for scenario in scenarios { - let description = scenario.description.as_deref().unwrap_or("").trim(); - let behavior = file_stem(&scenario.source_path) - .split_once("__") - .map(|(_, value)| value) - .unwrap_or(""); - if description.is_empty() || generic(description) || generic(behavior) { - faults.push(path_fault( - rule, - &scenario.source_path, - "scenario description or filename uses a generic case label".into(), - "Write a concrete business description and a __.sql filename." - .into(), - )); - } - } - faults -} - fn expected_parent( evaluation: &ProjectEvaluationRequest<'_>, test: &SqlTestFact, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 3451e6c91e..7a873ed3de 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -388,6 +388,7 @@ def _function_resource(function: CompiledFunction) -> dict[str, object]: "name": function.name, "relative_path": str(function.relative_path), "language": function.language.value, + "description": function.description, "return_kind": "table" if function.return_columns else "scalar", "returns": function.returns, "return_columns": [ diff --git a/src/sqlbuild/cli/commands/_helpers/playground/copy.py b/src/sqlbuild/cli/commands/_helpers/playground/copy.py index 76e3b3ca2b..cc0fd8c7c4 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/copy.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/copy.py @@ -56,6 +56,7 @@ _PYTHON_NODES_SOURCES_YML: str = """sources: - name: raw_orders + description: Raw orders loaded by the raw_orders Python loader managed: true write_strategy: table columns: @@ -70,6 +71,7 @@ """ _PYTHON_NODES_FACT_ORDERS_SQL: str = """MODEL ( + description "One row per order with its customer and amount", materialized table, columns ( order_id (nullable false, audits [not_null, unique]), diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql index 783d210d1a..d0ca81865b 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql @@ -1,4 +1,4 @@ -MODEL (materialized table); +MODEL (description "Revenue per customer from completed waffle orders", materialized table); SELECT customer_id, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql index c7e26d3d38..c743a3cb31 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql @@ -1,4 +1,4 @@ -MODEL (materialized table); +MODEL (description "One row per waffle order with its customer and line total", materialized table); SELECT o.order_id, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt index 43a81a385e..dc2c376c64 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt @@ -13,6 +13,8 @@ from sqlbuild.loaders import loader ], ) def fetch_order_events(ctx): + """Fetch new order events since the last loaded sequence number.""" + if ctx.current_cursor_value is None: next_seq = 1 else: @@ -47,6 +49,8 @@ def fetch_order_events(ctx): ], ) def fetch_customers(ctx): + """Fetch customer records changed since the last load.""" + if ctx.current_cursor_value is None: return [ {"customer_id": 1, "plan_name": "basic", "load_seq": 1}, @@ -68,6 +72,8 @@ def fetch_customers(ctx): ], ) def fetch_prices(ctx): + """Fetch the current waffle price list.""" + classic_price = 600 if ctx.current_cursor_value is None else 650 return [ {"waffle_type": "classic", "price_cents": classic_price, "load_seq": 1}, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml index be3905ca72..4e545e8468 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_orders + description: Raw waffle orders loaded by the Python loader contract: enforced managed: true write_strategy: table @@ -17,6 +18,7 @@ sources: - name: load_seq type: INTEGER - name: raw_customers + description: Raw customers loaded by the Python loader contract: enforced managed: true write_strategy: table diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql index 7f1bc18a47..a2633f291e 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Orders placed by one customer with their line totals", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql index 98ef857e40..a722c20280 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml index 3d5b2d89a9..9f97ecff49 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw__customers + description: Raw customer records loaded by the Python loader contract: enforced managed: true write_strategy: table @@ -16,6 +17,7 @@ sources: type: TIMESTAMP - name: raw__orders + description: Raw order records loaded by the Python loader contract: enforced managed: true write_strategy: delete_insert @@ -35,6 +37,7 @@ sources: type: VARCHAR - name: raw__payments + description: Payment records for each order, defined inline expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index bd2634be06..fefad914c0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -88,18 +88,14 @@ cursor_intrinsics_analysis_sql, get_validated_model_cursor_intrinsics, ) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_declaration_scope_resolver, -) +from sqlbuild.compiler.compile._helpers.render.declarations import build_declaration_scope_resolver from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros, find_macro_call_names, ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys -from sqlbuild.compiler.compile._helpers.sql_tests.identity import ( - build_sql_test_case_fingerprint, -) +from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, @@ -1690,6 +1686,7 @@ def _assemble_compiled_function( entry_point=function_input.entry_point, packages=function_input.packages, tags=function_input.tags, + description=function_input.description, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 32fc8f463c..b2737a43b7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -9,6 +9,7 @@ MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY, MEASUREMENT_THRESHOLDS_HEADER_KEY, ) +from sqlbuild.compiler.auditing.main._builtins import build_builtin_audit_resolution from sqlbuild.compiler.auditing.main._parse_audit_instance import ( parse_measurement_thresholds, parse_minimum_samples, @@ -119,6 +120,32 @@ def remember_scoped_declarations( self.scoped_declarations[key] = declarations +def build_project_audit_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + model_inputs: tuple[CompileModelInput, ...], + source_inputs: tuple[CompileSourceInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], +) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: + """Resolve built-in-aware generic audits and build every project audit input.""" + + generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] + diagnostics: tuple[CompilerDiagnostic, ...] + generic_audit_definitions, diagnostics = build_builtin_audit_resolution( + index_generic_audit_definitions(discovered_inputs.audit_files) + ) + audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( + discovered_inputs=discovered_inputs, + context=context, + model_inputs=model_inputs, + source_inputs=source_inputs, + generic_audit_definitions=generic_audit_definitions, + seed_inputs=seed_inputs, + ) + return audit_inputs, diagnostics + + def build_audit_inputs( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2b273a7ae..c2260c1862 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -170,7 +170,11 @@ _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_REUSABLE_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"columns"}) +_MODEL_HEADER_COLUMNS_KEY: str = "columns" +_MODEL_HEADER_DESCRIPTION_KEY: str = "description" +_REUSABLE_MODEL_HEADER_KEYS: frozenset[str] = frozenset( + {_MODEL_HEADER_COLUMNS_KEY, _MODEL_HEADER_DESCRIPTION_KEY} +) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} ) @@ -286,6 +290,9 @@ class _ReusableModelConfigCache: path_defaults: dict[str, dict[str, object]] target_config: TargetConfig | None reusable_by_path_default: dict[str | None, bool] = field(default_factory=dict) + inherited_values_by_path_default: dict[str | None, dict[str, object]] = field( + default_factory=dict + ) reusable_metadata_by_identity: dict[int, tuple[object, bool]] = field(default_factory=dict) configs: dict[str | None, CompileModelConfig] = field(default_factory=dict) @@ -315,9 +322,15 @@ def remember( reusable_metadata: dict[str, object] | None = self._reusable_metadata(model_header_values) if reusable_metadata is None or not self._is_reusable(matched_path_default): return + inherited_values: dict[str, object] = self.inherited_values_by_path_default[ + matched_path_default + ] reusable_values: dict[str, object] = { key: value for key, value in config.values.items() if key not in reusable_metadata } + reusable_values.update( + (key, inherited_values[key]) for key in reusable_metadata if key in inherited_values + ) self.configs[matched_path_default] = replace(config, values=reusable_values) def _reusable_metadata( @@ -326,9 +339,15 @@ def _reusable_metadata( ) -> dict[str, object] | None: if not model_header_values: return {} - if model_header_values.keys() != _REUSABLE_MODEL_HEADER_KEYS: + if not model_header_values.keys() <= _REUSABLE_MODEL_HEADER_KEYS: + return None + if _contains_dynamic_or_unsupported_reusable_metadata( + model_header_values.get(_MODEL_HEADER_DESCRIPTION_KEY) + ): return None - metadata: object = model_header_values["columns"] + if _MODEL_HEADER_COLUMNS_KEY not in model_header_values: + return model_header_values + metadata: object = model_header_values[_MODEL_HEADER_COLUMNS_KEY] cached: tuple[object, bool] | None = self.reusable_metadata_by_identity.get(id(metadata)) if cached is not None and cached[0] is metadata: return model_header_values if cached[1] else None @@ -356,6 +375,7 @@ def _is_reusable(self, matched_path_default: str | None) -> bool: (layered_values, target_namespace_values) ) and not _contains_mutable_nested_config(layered_values) self.reusable_by_path_default[matched_path_default] = reusable + self.inherited_values_by_path_default[matched_path_default] = layered_values return reusable diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 014b743b52..03fc4cbf80 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -262,6 +262,11 @@ def build_sql_function_inputs( relative_path=function_file.relative_path, language="SQL", ), + description=_parse_function_description( + header_values=header_values, + relative_path=function_file.relative_path, + language="SQL", + ), declaration_usages=expansion.usages, ) ) @@ -442,6 +447,11 @@ def _build_python_function_input( relative_path=python_function_file.relative_path, language="Python", ), + description=_parse_function_description( + header_values=header_values, + relative_path=python_function_file.relative_path, + language="Python", + ), ) @@ -590,6 +600,19 @@ def _parse_python_packages(*, raw_packages: object | None, relative_path: Path) return tuple(packages) +def _parse_function_description( + *, header_values: dict[str, object], relative_path: Path, language: str +) -> str | None: + raw_description: object | None = header_values.get("description") + if raw_description is None: + return None + if not isinstance(raw_description, str): + raise CompileInputError( + f"{language} function file {relative_path} description must be a string" + ) + return raw_description.strip() + + def _parse_function_tags( *, header_values: dict[str, object], relative_path: Path, language: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 499df98900..fc5f99faaf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -53,6 +53,8 @@ ) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) +_MODEL_DESCRIPTION_KEY: str = "description" +_MODEL_SCHEMA_KEY: str = "model_schema" def _contains_template_data_cached(*, value: object, cache: IdentityPresenceCache | None) -> bool: @@ -531,6 +533,36 @@ def _resolve_model_schema( return declaration +def has_resolved_model_description( + *, + model_file: DiscoveredSqlModelFile, + defaults: DefaultsConfig, + path_defaults: dict[str, dict[str, object]], + model_schemas: dict[str, ModelSchemaDeclaration], +) -> bool: + """Return whether layered config or the model schema gives the model a description.""" + + values: dict[str, object] = build_layered_model_values( + defaults=defaults, + path_defaults=path_defaults, + matched_path_default=find_matching_path_default( + model_file=model_file, path_defaults=path_defaults + ), + model_header_values=model_file.header_values, + ) + schema_name: object | None = values.get(_MODEL_SCHEMA_KEY) + schema: ModelSchemaDeclaration | None = ( + model_schemas.get(schema_name) if isinstance(schema_name, str) else None + ) + return _is_non_empty_text(values.get(_MODEL_DESCRIPTION_KEY)) or _is_non_empty_text( + schema.description if schema is not None else None + ) + + +def _is_non_empty_text(value: object | None) -> bool: + return isinstance(value, str) and bool(value.strip()) + + def find_matching_path_default( *, model_file: DiscoveredSqlModelFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py new file mode 100644 index 0000000000..ec08682bef --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py @@ -0,0 +1,458 @@ +"""Require a non-empty description on every named project resource.""" + +from __future__ import annotations + +import ast +import inspect +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from functools import partial +from pathlib import Path + +from sqlbuild.compiler.compile.constants import MISSING_DESCRIPTION_CODE +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlFunctionInput, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, + FunctionLanguage, +) +from sqlbuild.compiler.discovery.constants import PYTHON_UDF_DECORATOR_NAME, YAML_FILE_SUFFIXES +from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line +from sqlbuild.compiler.discovery.models import ( + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + DiscoveredProvider, +) +from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment +from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.snippet_help import snippet_help +from sqlbuild.spec.contracts.models import SourceLocation + +_LINE_COMMENT_START: str = "--" +_BLOCK_COMMENT_START: str = "/*" + + +@dataclass(frozen=True) +class _Resource: + """One named resource and the description it declares, if any.""" + + kind: str + name: str + description: str | None + location: Callable[[], SourceLocation] + help: str + resource_type: CompiledResourceType | None = None + + +@dataclass(frozen=True) +class _PythonNode: + """The shared shape of a decorated Python node.""" + + kind: str + decorator: str + name: str + description: str | None + relative_path: Path + function: Callable[..., object] + + +def missing_description_diagnostics( + *, + discovered_inputs: DiscoveredProjectInputs, + model_inputs: tuple[CompileModelInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], + source_inputs: tuple[CompileSourceInput, ...], + function_inputs: tuple[CompileSqlFunctionInput, ...], +) -> tuple[CompilerDiagnostic, ...]: + """Return one error for each named resource without a non-empty description.""" + + loaders_by_name: dict[str, DiscoveredLoaderFunction] = { + loader.name: loader for loader in discovered_inputs.loader_functions + } + resources: tuple[_Resource, ...] = ( + *(_model(model_input) for model_input in model_inputs), + *_scenarios(discovered_inputs), + *(_seed(seed_input) for seed_input in seed_inputs), + *( + _source(source_input=source_input, loaders_by_name=loaders_by_name) + for source_input in source_inputs + ), + *(_function(function_input) for function_input in function_inputs), + *_sql_hooks(discovered_inputs), + *( + _python_resource(node) + for node in _python_nodes( + discovered_inputs=discovered_inputs, source_inputs=source_inputs + ) + ), + *(_provider(provider) for provider in discovered_inputs.providers), + ) + return tuple( + _diagnostic(resource) + for resource in resources + if resource.description is None or not resource.description.strip() + ) + + +def _diagnostic(resource: _Resource) -> CompilerDiagnostic: + state: str = ( + "has no description" if resource.description is None else "has an empty description" + ) + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=MISSING_DESCRIPTION_CODE, + message=f"{resource.kind} '{resource.name}' {state}", + resource_type=resource.resource_type, + resource_name=resource.name, + location=resource.location(), + help=resource.help, + ) + + +def _model(model_input: CompileModelInput) -> _Resource: + name: str = model_input.model_file.file_path.stem + path: Path = model_input.model_file.relative_path + contents: str = model_input.model_file.contents + help_text: str = _header_help( + kind="model", + name=name, + statement="MODEL", + entry=f'description "What one row of {name} represents"', + path=path, + ) + schema_entry_description: str | None = ( + model_input.schema_entry.description if model_input.schema_entry is not None else None + ) + return _Resource( + kind="model", + name=name, + description=schema_entry_description, + location=partial(_header_location, path=path, contents=contents), + help=help_text, + resource_type=CompiledResourceType.MODEL, + ) + + +def _scenarios(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: + for scenario_file in discovered_inputs.scenario_files: + raw_description: object | None = scenario_file.header_values.get("description") + yield _Resource( + kind="scenario", + name=scenario_file.name, + description=raw_description if isinstance(raw_description, str) else None, + location=partial( + _header_location, + path=scenario_file.relative_path, + contents=scenario_file.contents, + ), + help=_header_help( + kind="scenario", + name=scenario_file.name, + statement="SCENARIO", + entry=f'description "The behaviour {scenario_file.name} proves"', + path=scenario_file.relative_path, + ), + resource_type=CompiledResourceType.SQL_SCENARIO, + ) + + +def _seed(seed_input: CompileSeedInput) -> _Resource: + name: str = seed_input.schema_entry.name + return _Resource( + kind="seed", + name=name, + description=seed_input.schema_entry.description, + location=partial( + _yaml_location, + path=seed_input.schema_file.relative_path, + contents=seed_input.schema_file.contents, + name=name, + ), + help=_yaml_help(kind="seed", name=name, path=seed_input.schema_file.relative_path), + resource_type=CompiledResourceType.SEED, + ) + + +def _source( + *, source_input: CompileSourceInput, loaders_by_name: dict[str, DiscoveredLoaderFunction] +) -> _Resource: + name: str = source_input.source_entry.name + loader: DiscoveredLoaderFunction | None = ( + loaders_by_name.get(source_input.source_entry.loader) + if source_input.source_entry.loader is not None + else None + ) + description: str | None = source_input.source_entry.description + if (description is None or not description.strip()) and loader is not None: + description = loader.description or description + help_text: str = _yaml_help( + kind="source", name=name, path=source_input.source_file.relative_path + ) + if loader is not None and loader.relative_path.suffix not in YAML_FILE_SUFFIXES: + help_text = join_helps( + help_text, + f"or describe its loader '{loader.name}' in {loader.relative_path.as_posix()} " + f'with a docstring or @loader(description="What one row of {name} represents")', + ) + return _Resource( + kind="source", + name=name, + description=description, + location=partial( + _yaml_location, + path=source_input.source_file.relative_path, + contents=source_input.source_file.contents, + name=name, + ), + help=help_text, + resource_type=CompiledResourceType.SOURCE, + ) + + +def _function(function_input: CompileSqlFunctionInput) -> _Resource: + name: str = function_input.name + path: Path = function_input.function_file.relative_path + resource_type: CompiledResourceType = ( + CompiledResourceType.TABLE_FN if function_input.return_columns else CompiledResourceType.UDF + ) + location: Callable[[], SourceLocation] + help_text: str + if function_input.language == FunctionLanguage.PYTHON: + location = partial( + _decorator_location, + path=path, + contents=function_input.function_file.contents, + decorator=PYTHON_UDF_DECORATOR_NAME, + ) + help_text = snippet_help( + purpose=f"to describe function '{name}'", + target=f"give its function in {path.as_posix()} a docstring or add this to @udf", + lines=(f'@udf(description="What {name} returns", ...)',), + ) + else: + location = partial( + _header_location, path=path, contents=function_input.function_file.contents + ) + help_text = _header_help( + kind="function", + name=name, + statement="FUNCTION", + entry=f'description "What {name} returns"', + path=path, + ) + return _Resource( + kind="function", + name=name, + description=function_input.description, + location=location, + help=help_text, + resource_type=resource_type, + ) + + +def _sql_hooks(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: + for hook_file in discovered_inputs.sql_hook_files: + yield _Resource( + kind="hook", + name=hook_file.name, + description=hook_file.description, + location=partial( + _header_location, path=hook_file.relative_path, contents=hook_file.contents + ), + help=_header_help( + kind="hook", + name=hook_file.name, + statement="HOOK", + entry=f'description "What {hook_file.name} does"', + path=hook_file.relative_path, + description_only=True, + ), + ) + + +def _python_nodes( + *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[CompileSourceInput, ...] +) -> Iterator[_PythonNode]: + source_owned_loaders: frozenset[str] = frozenset( + source_input.source_entry.loader + for source_input in source_inputs + if source_input.source_entry.loader is not None + ) + source_paths: frozenset[Path] = frozenset( + source_file.relative_path for source_file in discovered_inputs.source_files + ) + for loader in discovered_inputs.loader_functions: + if loader.name in source_owned_loaders or loader.relative_path in source_paths: + continue + yield _PythonNode( + kind="loader", + decorator="loader", + name=loader.name, + description=loader.description, + relative_path=loader.relative_path, + function=loader.function, + ) + for kind, functions in ( + ("task", discovered_inputs.task_functions), + ("asset", discovered_inputs.asset_functions), + ("check", discovered_inputs.check_functions), + ("hook", discovered_inputs.hook_functions), + ): + for function in functions: + yield _PythonNode( + kind=kind, + decorator=kind, + name=function.name, + description=function.description, + relative_path=function.relative_path, + function=function.function, + ) + + +def _python_resource(node: _PythonNode) -> _Resource: + return _Resource( + kind=node.kind, + name=node.name, + description=node.description, + location=partial(_function_location, path=node.relative_path, function=node.function), + help=snippet_help( + purpose=f"to describe {node.kind} '{node.name}'", + target=( + f"give its function in {node.relative_path.as_posix()} a docstring " + f"or pass a description to @{node.decorator}" + ), + lines=(f'@{node.decorator}(description="What {node.name} does", ...)',), + ), + ) + + +def _provider(provider: DiscoveredProvider) -> _Resource: + class_name: str = provider.provider_class.__name__ + raw_doc: object | None = provider.provider_class.__dict__.get("__doc__") + return _Resource( + kind="provider", + name=provider.name, + description=inspect.cleandoc(raw_doc) if isinstance(raw_doc, str) else None, + location=partial( + _class_location, path=provider.relative_path, provider_class=provider.provider_class + ), + help=snippet_help( + purpose=f"to describe provider '{provider.name}'", + target=f"add a class docstring in {provider.relative_path.as_posix()}", + lines=( + f"class {class_name}(Provider):", + f' """What {provider.name} provides."""', + ), + ), + ) + + +def _header_help( + *, + kind: str, + name: str, + statement: str, + entry: str, + path: Path, + description_only: bool = False, +) -> str: + lines: tuple[str, ...] = ( + (f"{statement} ({entry});",) + if description_only + else (f"{statement} (", f" {entry},", " ...", ");") + ) + return snippet_help( + purpose=f"to describe {kind} '{name}'", + target=f"add this to the {statement} header in {path.as_posix()}", + lines=lines, + ) + + +def _yaml_help(*, kind: str, name: str, path: Path) -> str: + return snippet_help( + purpose=f"to describe {kind} '{name}'", + target=f"add `description` to its entry in {path.as_posix()}", + lines=(f"- name: {name}", f" description: What one row of {name} represents"), + ) + + +def _header_location(*, path: Path, contents: str) -> SourceLocation: + """Locate the statement header: the first code after leading whitespace and comments.""" + + position: int = 0 + while True: + while position < len(contents) and contents[position].isspace(): + position += 1 + if contents.startswith(_LINE_COMMENT_START, position): + position = skip_line_comment(sql=contents, start=position) + elif contents.startswith(_BLOCK_COMMENT_START, position): + position = skip_block_comment(sql=contents, start=position) + else: + return _offset_location(path=path, contents=contents, offset=position) + + +def _function_location(*, path: Path, function: Callable[..., object]) -> SourceLocation: + return SourceLocation(path=path, line=_function_line(function), column=1) + + +def _class_location(*, path: Path, provider_class: type[object]) -> SourceLocation: + return SourceLocation(path=path, line=_class_line(provider_class), column=1) + + +def _yaml_location(*, path: Path, contents: str, name: str) -> SourceLocation: + line: int | None = yaml_entry_line(contents=contents, name=name) + return SourceLocation(path=path, line=line or 1, column=1) + + +def _decorator_location(*, path: Path, contents: str, decorator: str) -> SourceLocation: + """Locate the `@decorator` line of the decorated function, ignoring docstrings and comments.""" + + try: + module: ast.Module = ast.parse(contents) + except SyntaxError: + return SourceLocation(path=path, line=1, column=1) + for node in ast.walk(module): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + for expression in node.decorator_list: + if _decorator_name(expression) == decorator: + return SourceLocation( + path=path, line=expression.lineno, column=expression.col_offset + ) + return SourceLocation(path=path, line=1, column=1) + + +def _decorator_name(expression: ast.expr) -> str | None: + target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression + if isinstance(target, ast.Name): + return target.id + if isinstance(target, ast.Attribute): + return target.attr + return None + + +def _offset_location(*, path: Path, contents: str, offset: int) -> SourceLocation: + line: int = contents.count("\n", 0, offset) + 1 + column: int = offset - (contents.rfind("\n", 0, offset) + 1) + 1 + return SourceLocation(path=path, line=line, column=column) + + +def _function_line(function: Callable[..., object]) -> int: + code: object | None = getattr(inspect.unwrap(function), "__code__", None) + line: object | None = getattr(code, "co_firstlineno", None) + return line if isinstance(line, int) else 1 + + +def _class_line(provider_class: type[object]) -> int: + try: + return inspect.getsourcelines(provider_class)[1] + except (OSError, TypeError): + return 1 diff --git a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py new file mode 100644 index 0000000000..5a42738641 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py @@ -0,0 +1,41 @@ +"""Compile-equivalent description resolution for header-only tooling.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.model_config import ( + has_resolved_model_description, +) +from sqlbuild.compiler.compile._helpers.render.declarations import ( + build_public_model_schema_index, +) +from sqlbuild.compiler.discovery.main._model_description_inputs import ( + discover_model_description_inputs, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, ModelSchemaDeclaration + + +class ModelDescriptionResolution: + """Resolve model descriptions that come from outside the MODEL header.""" + + @staticmethod + def externally_described_models(*, project_dir: Path) -> frozenset[Path]: + """Return model files that `[defaults]`, `[path_defaults]` or a model schema describe.""" + + discovered_inputs: DiscoveredProjectInputs = discover_model_description_inputs( + project_dir=project_dir + ) + model_schemas: dict[str, ModelSchemaDeclaration] = build_public_model_schema_index( + discovered_inputs=discovered_inputs + ) + return frozenset( + model_file.file_path + for model_file in discovered_inputs.model_files + if has_resolved_model_description( + model_file=model_file, + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + model_schemas=model_schemas, + ) + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 73d04a6510..063b1f129f 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -15,6 +15,39 @@ HOOK_READS_OWN_DEPENDANT_CODE: str = "P007" HARD_CODED_PROJECT_RELATION_CODE: str = "P008" UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" +MISSING_DESCRIPTION_CODE: str = "P010" +DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { + "model_files": "model", + "scenario_files": "scenario", + "seed_files": "seed", + "source_files": "source", + "sql_function_files": "function", + "python_function_files": "function", + "sql_hook_files": "hook", + "hook_functions": "hook", + "loader_functions": "loader", + "task_functions": "task", + "asset_functions": "asset", + "check_functions": "check", + "providers": "provider", +} +DESCRIPTION_EXEMPT_INPUTS: dict[str, str] = { + "project_config": "project configuration, not a named resource", + "local_config": "local configuration, not a named resource", + "project_dir": "project location, not a named resource", + "adapter_file": "project adapter override, not a named resource", + "enum_files": "typed value declarations used inside SQL, not graph resources", + "constant_files": "typed value declarations used inside SQL, not graph resources", + "macro_files": "SQL helpers expanded inline, not graph resources", + "model_schema_files": "shared column schemas; models that use one are checked", + "schema_files": "seed YAML declarations; each declared seed is checked", + "test_files": "SQL unit tests are named by their subject and expected behaviour", + "audit_files": "audits are assertions named by what they check", + "audit_factories": "generate audits, which are assertions", + "materialization_files": "materialization strategies, not graph resources", + "event_exporters": "runtime event outputs, not graph resources", + "command_output_sinks": "runtime command outputs, not graph resources", +} SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" MODEL_DIRECTORY_NAME: str = "models" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 534c86793f..0ffd4eb089 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -5,11 +5,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild.compiler.auditing.main._builtins import build_builtin_audit_resolution -from sqlbuild.compiler.compile._helpers.attachment.audits import ( - build_audit_inputs, - index_generic_audit_definitions, -) +from sqlbuild.compiler.compile._helpers.attachment.audits import build_project_audit_inputs from sqlbuild.compiler.compile._helpers.attachment.core import ( build_effective_connection, build_effective_settings, @@ -34,6 +30,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( with_collected_compile_diagnostics, ) +from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( + missing_description_diagnostics, +) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -61,8 +60,6 @@ ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredProjectInputs, EnumDeclaration, ModelSchemaDeclaration, @@ -191,23 +188,26 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) - project_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = ( - index_generic_audit_definitions(discovered_inputs.audit_files) - ) - generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] - diagnostics: tuple[CompilerDiagnostic, ...] - generic_audit_definitions, diagnostics = build_builtin_audit_resolution( - project_audit_definitions - ) - diagnostics = (*diagnostics, *model_build.diagnostics) - audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( + audit_inputs: tuple[CompileAuditInput, ...] + audit_diagnostics: tuple[CompilerDiagnostic, ...] + audit_inputs, audit_diagnostics = build_project_audit_inputs( discovered_inputs=discovered_inputs, context=model_context, model_inputs=model_build.inputs, source_inputs=source_inputs, - generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, ) + diagnostics: tuple[CompilerDiagnostic, ...] = ( + *audit_diagnostics, + *model_build.diagnostics, + *missing_description_diagnostics( + discovered_inputs=discovered_inputs, + model_inputs=model_build.inputs, + seed_inputs=seed_inputs, + source_inputs=source_inputs, + function_inputs=sql_function_inputs, + ), + ) return CompileProjectInputs( project_config=discovered_inputs.project_config, local_config=discovered_inputs.local_config, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 282597ae36..93bfc4495e 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -738,6 +738,7 @@ class CompileSqlFunctionInput: packages: tuple[str, ...] = field(default_factory=tuple) declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) tags: tuple[str, ...] = field(default_factory=tuple) + description: str | None = None @dataclass(frozen=True) @@ -980,6 +981,7 @@ class CompiledFunction: entry_point: str | None = None packages: tuple[str, ...] = field(default_factory=tuple) tags: tuple[str, ...] = field(default_factory=tuple) + description: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index da75d7a2b3..0abb5c741f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -1658,6 +1658,7 @@ def _append_python_node_function( unique_key=loader_definition.unique_key, columns=loader_definition.columns, contract=loader_definition.contract, + description=loader_definition.description, provider_usages=_provider_usages( function=function, provider_by_name=resolved_provider_by_name, diff --git a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py index c6f1d2e12b..7a01646d0e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py +++ b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py @@ -50,7 +50,15 @@ {"config", "destination", "resources", "schema", "type"} ) _DLT_RESOURCE_COMMON_KEYS: frozenset[str] = frozenset( - {"incremental", "merge_key", "name", "primary_key", "schema", "write_disposition"} + { + "description", + "incremental", + "merge_key", + "name", + "primary_key", + "schema", + "write_disposition", + } ) _DLT_RESOURCE_TYPE_KEYS: dict[str, frozenset[str]] = { DLT_SOURCE_TYPE_FILESYSTEM: frozenset({"reader"}), @@ -338,6 +346,13 @@ def _parse_dlt_resource_entry( ) return SourceEntry( name=name, + description=optional_non_empty_string( + entry=resource, + key="description", + file_path=file_path, + label=f"dlt resource '{name}'", + error_class=SourceParseError, + ), schema=schema, table=name, managed=True, @@ -426,6 +441,7 @@ def _raw_dlt_resource_config(*, source_type: str, resource: dict[str, object]) - "merge_key", "incremental", "schema", + "description", } if source_type == DLT_SOURCE_TYPE_REST_API: return {"endpoint": resource["endpoint"]} diff --git a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py index b4814b41bf..80493330ba 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py +++ b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py @@ -13,6 +13,8 @@ ) from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError +_UDF_DESCRIPTION_KEY: str = "description" + def parse_python_function(*, contents: str, file_path: Path) -> tuple[dict[str, object], str, str]: """Parse one @udf-decorated Python function without importing project code.""" @@ -75,6 +77,9 @@ def parse_python_function(*, contents: str, file_path: Path) -> tuple[dict[str, error_class=ModelSqlParseError, ) + docstring: str | None = ast.get_docstring(function_node) + if _UDF_DESCRIPTION_KEY not in values and docstring is not None: + values[_UDF_DESCRIPTION_KEY] = docstring entry_point: object | None = values.get("entry_point") if entry_point is None: values["entry_point"] = function_node.name diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 5777e15c13..2d2396b48f 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -18,7 +18,7 @@ PYTHON_NODE_ROOT: str = "python" PYTHON_UDF_DECORATOR_NAME: str = "udf" SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( - {"arguments", "returns", "database", "schema", "tags"} + {"arguments", "returns", "database", "schema", "tags", "description"} ) SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( { @@ -94,6 +94,7 @@ REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) PYTHON_UDF_KEYS: frozenset[str] = frozenset( { + "description", "arguments", "returns", "runtime_version", diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py new file mode 100644 index 0000000000..5925541c97 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -0,0 +1,37 @@ +"""SQL-only discovery of the inputs that decide a model's description.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_model_files, + discover_model_schema_files, +) +from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault +from sqlbuild.spec.contracts.models import LocalConfig + + +def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: + """Load project config, model files and model schemas without importing project Python.""" + + return DiscoveredProjectInputs( + project_config=load_project_config(project_dir=project_dir), + local_config=LocalConfig(), + model_files=discover_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=False, + extract_output_column_locations=False, + on_fault=_skip_unreadable_file, + ), + model_schema_files=discover_model_schema_files( + project_dir=project_dir, on_fault=_skip_unreadable_file + ), + ) + + +def _skip_unreadable_file(fault: DiscoveryFileFault) -> None: + """Leave an unparseable file unresolved; format reports that file's own faults.""" + + del fault diff --git a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py new file mode 100644 index 0000000000..375abe4f80 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py @@ -0,0 +1,24 @@ +"""Locate one named entry in an authored YAML declaration file.""" + +from __future__ import annotations + +import re + + +def yaml_entry_line(*, contents: str, name: str) -> int | None: + """Return the 1-based line of the outermost `- name: ` entry, if present.""" + + pattern: re.Pattern[str] = re.compile( + rf"^(?P\s*)-\s*name:\s*(?P['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$" + ) + best: tuple[int, int] | None = None + index: int + line: str + for index, line in enumerate(contents.splitlines(), start=1): + match: re.Match[str] | None = pattern.match(line) + if match is None: + continue + indent: int = len(match.group("indent")) + if best is None or indent < best[0]: + best = (indent, index) + return None if best is None else best[1] diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 4f6f0a5ab7..a43a6f99f9 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -436,6 +436,7 @@ class DiscoveredLoaderFunction: unique_key: tuple[str, ...] = field(default_factory=tuple) columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) contract: str | None = None + description: str | None = None connection_mode: LoaderConnectionMode = LoaderConnectionMode.SQLBUILD provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py index f9030a58ec..77d2a32c58 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from dataclasses import dataclass from functools import partial from typing import Any @@ -13,6 +12,7 @@ from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line from sqlbuild.compiler.planner._helpers.resolve.sources import render_source_read_relation from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope @@ -172,22 +172,5 @@ def _missing_source_tables_help(*, missing: tuple[_MissingSourceTable, ...]) -> def _declaration_location(source: CompiledSource) -> str: path: str = source.source_file.relative_path.as_posix() - line: int | None = _declaration_line(contents=source.source_file.contents, name=source.name) + line: int | None = yaml_entry_line(contents=source.source_file.contents, name=source.name) return path if line is None else f"{path}:{line}" - - -def _declaration_line(*, contents: str, name: str) -> int | None: - pattern: re.Pattern[str] = re.compile( - rf"^(?P\s*)-\s*name:\s*(?P['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$" - ) - best: tuple[int, int] | None = None - index: int - line: str - for index, line in enumerate(contents.splitlines(), start=1): - match: re.Match[str] | None = pattern.match(line) - if match is None: - continue - indent: int = len(match.group("indent")) - if best is None or indent < best[0]: - best = (indent, index) - return None if best is None else best[1] diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py index 69090a97d3..5b7b95f5ac 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py @@ -224,6 +224,7 @@ def _build_loader_node( else None, }, ), + description=loader.description, loader=DiscoveredPythonLoaderMetadata( destination=loader.destination, write_strategy=loader.write_strategy, diff --git a/src/sqlbuild/errors/setting_help/_helpers/text.py b/src/sqlbuild/errors/setting_help/_helpers/text.py index cded5445f0..3bbe56e013 100644 --- a/src/sqlbuild/errors/setting_help/_helpers/text.py +++ b/src/sqlbuild/errors/setting_help/_helpers/text.py @@ -61,5 +61,10 @@ def model_header_help(*, purpose: str, entry: str, follow_up: str | None) -> str return "\n".join(lines) +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: + snippet: str = "\n".join(f"{SETTING_SNIPPET_INDENT}{line}" for line in lines) + return f"{purpose}, {target}:\n{snippet}" + + def join_helps(helps: tuple[str, ...]) -> str: return ADDITIONAL_HELP_SEPARATOR.join(helps) diff --git a/src/sqlbuild/errors/setting_help/main/snippet_help.py b/src/sqlbuild/errors/setting_help/main/snippet_help.py new file mode 100644 index 0000000000..6f5025d24a --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/snippet_help.py @@ -0,0 +1,11 @@ +"""Public entry for help that shows an exact authored snippet to add.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import snippet_help as _snippet_help + + +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: + """Render `, :` followed by the exact lines to write, one per line.""" + + return _snippet_help(purpose=purpose, target=target, lines=lines) diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 9855d1123e..ed33796a3d 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -8,6 +8,9 @@ from dataclasses import dataclass from pathlib import Path +from sqlbuild.compiler.compile.classes.model_description_resolution import ( + ModelDescriptionResolution, +) from sqlbuild.compiler.discovery._helpers.sql.model_files import ( parse_header_values, # noqa: FFL102 - the compiler owns the canonical native header grammar prepare_model_header_tokens, @@ -32,7 +35,6 @@ VIOLATION_SEVERITY_WARNING, ) from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation -from sqlbuild.lint.types import LintSeverity _QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) _ESCAPE_CHARACTER: str = "\\" @@ -81,7 +83,6 @@ def lint_native_headers( file_path: Path, headers: tuple[HeaderSpan, ...], config: LintConfig, - description_present_severity: LintSeverity = VIOLATION_SEVERITY_FAULT, ) -> tuple[LintViolation, ...]: """Run all native header rules and return their violations.""" @@ -96,7 +97,6 @@ def lint_native_headers( header=header, config=config, line_starts=line_starts, - description_present_severity=description_present_severity, ) ) violations.extend( @@ -176,7 +176,6 @@ def _header_parse_faults( file_path=file_path, headers=headers, config=config, - description_present_severity=VIOLATION_SEVERITY_WARNING, ) if violation.code == RULE_HEADER_PARSE ) @@ -230,7 +229,6 @@ def _lint_header_values( header: HeaderSpan, config: LintConfig, line_starts: tuple[int, ...], - description_present_severity: LintSeverity, ) -> tuple[LintViolation, ...]: header_text: str = contents[header.start : header.end] try: @@ -276,8 +274,10 @@ def _lint_header_values( header=header, code=RULE_DESCRIPTION_PRESENT, message=f"{header.kind}() header requires a description", - remediation=f"Add a description to the {header.kind}() header.", - severity=description_present_severity, + remediation=( + f'Add description "" to the {header.kind}() header; ' + "every named resource must be described." + ), ) ) if header.kind in DESCRIPTION_HEADER_KINDS and isinstance(effective_description, str): @@ -698,7 +698,6 @@ def _violation_for_header_start( code: str, message: str, remediation: str, - severity: LintSeverity = VIOLATION_SEVERITY_FAULT, ) -> LintViolation: position: tuple[int, int] = _offset_to_position( offset=header.start, line_starts=_line_starts(contents) @@ -709,7 +708,7 @@ def _violation_for_header_start( column=position[1], code=code, message=message, - severity=severity, + severity=VIOLATION_SEVERITY_FAULT, engine=LINT_ENGINE_SQLBUILD, remediation=remediation, ) @@ -839,3 +838,22 @@ def _audits_key_start(*, text: str, bracket: int) -> int: while start > 0 and text[start - 1].isspace(): start -= 1 return max(0, start - len(_AUDITS_KEYWORD)) + + +def drop_resolved_description_faults( + *, violations: list[LintViolation], project_dir: Path +) -> list[LintViolation]: + """Drop `description-present` faults for models described outside their header.""" + + if not any(violation.code == RULE_DESCRIPTION_PRESENT for violation in violations): + return violations + described: frozenset[Path] = frozenset( + path.resolve() + for path in ModelDescriptionResolution.externally_described_models(project_dir=project_dir) + ) + return [ + violation + for violation in violations + if violation.code != RULE_DESCRIPTION_PRESENT + or violation.file_path.resolve() not in described + ] diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 9ae2581a3f..ebf419e223 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -37,7 +37,9 @@ IDENTIFIER_SEPARATOR_CHARACTER: str = "_" DESCRIPTION_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_MODEL, HEADER_KIND_SCENARIO}) -DESCRIPTION_REQUIRED_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_MODEL}) +DESCRIPTION_REQUIRED_HEADER_KINDS: frozenset[str] = frozenset( + {HEADER_KIND_MODEL, HEADER_KIND_SCENARIO, HEADER_KIND_FUNCTION, HEADER_KIND_HOOK} +) DEFAULT_MAX_DESCRIPTION_LINES: int = 10 DEFAULT_LINE_WIDTH: int = 100 diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index 91075f5b48..5bccae6650 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -11,6 +11,7 @@ from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import ( + drop_resolved_description_faults, format_native_headers, lint_native_headers, prepare_native_header_cache, @@ -20,7 +21,6 @@ from sqlbuild.lint._helpers.native_format import format_native_sql_bodies from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions -from sqlbuild.lint.constants import VIOLATION_SEVERITY_WARNING from sqlbuild.lint.exceptions import LintError from sqlbuild.lint.models import ( FormatChange, @@ -90,6 +90,7 @@ def run_format( ) ) violations.extend(format_faults) + violations = drop_resolved_description_faults(violations=violations, project_dir=project_dir) final_contents: dict[Path, str] = { path: updated_contents.get(path, contents) for path, contents in files.items() } @@ -214,7 +215,6 @@ def _lint_final_contents( file_path=file_path, headers=headers, config=config, - description_present_severity=VIOLATION_SEVERITY_WARNING, ) ) return violations diff --git a/src/sqlbuild/loaders.pyi b/src/sqlbuild/loaders.pyi index 1c7bd723d1..977f5e3c38 100644 --- a/src/sqlbuild/loaders.pyi +++ b/src/sqlbuild/loaders.pyi @@ -18,5 +18,6 @@ def loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[[Callable[..., object]], Callable[..., object]]: ... def get_loader_definition(function: Callable[..., object]) -> LoaderDefinition | None: ... diff --git a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py index c602b0ee3c..458112ffa8 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py @@ -23,6 +23,7 @@ def loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Mark a Python function as a SQLBuild source loader.""" @@ -36,6 +37,7 @@ def loader( unique_key=unique_key, columns=columns, contract=contract, + description=description, ) diff --git a/src/sqlbuild/python_nodes/main/_apply_loader.py b/src/sqlbuild/python_nodes/main/_apply_loader.py index c5def399e9..f2fc9f444f 100644 --- a/src/sqlbuild/python_nodes/main/_apply_loader.py +++ b/src/sqlbuild/python_nodes/main/_apply_loader.py @@ -8,6 +8,7 @@ from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_loader_dependencies +from sqlbuild.python_nodes._helpers.description_resolution import resolve_description from sqlbuild.python_nodes._helpers.loader_normalization import ( normalize_unique_key, normalize_write_strategy, @@ -28,6 +29,7 @@ def apply_loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild loader metadata to a Python function.""" @@ -44,6 +46,7 @@ def decorate(inner: Callable[..., object]) -> Callable[..., object]: unique_key=normalize_unique_key(unique_key), columns=normalize_columns(columns), contract=contract, + description=resolve_description(function=inner, description=description), ) return attach_definition( function=inner, diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index ede8bc1cae..f297bdd362 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -106,6 +106,7 @@ class LoaderDefinition: unique_key: tuple[str, ...] = () columns: tuple[SourceColumnEntry, ...] = () contract: str | None = None + description: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py index 4f2fddc868..10a496c931 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py @@ -145,7 +145,6 @@ def _rule_example(*, rule: Rule) -> str: "SQBRTEST102": "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", "SQBRTEST103": "models/staging/stg_orders.sql maps to tests/unit/staging/.", "SQBRTEST104": 'TEST (name "stg_orders__excludes_cancelled_orders");', - "SQBRTEST105": 'SCENARIO (description "Daily revenue includes successful payments");', "SQBRTEST201": "Attach not_null/unique audits to the model key.", } if rule.guidance is not None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index cbe17445f1..98a0dbe6ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -496,3 +496,46 @@ class UnionFixtureCompileScalingTestCase: expected_sql_tests: int expected_errors: int expected_max_scaling_ratio: float + + +@dataclass(frozen=True) +class RequiredDescriptionCase: + description: str + missing_files: tuple[tuple[str, str], ...] + described_files: tuple[tuple[str, str], ...] + expected_diagnostic: tuple[str, str, int] + expected_message: str + expected_help_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RequiredDescriptionAggregateCase: + description: str + files: tuple[tuple[str, str], ...] + expected_codes: frozenset[str] + expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class RequiredDescriptionPlanCase: + description: str + model_sql: str + expected_returncode: int + expected_stderr_fragment: str + + +@dataclass(frozen=True) +class PathDefaultDescriptionCase: + description: str + model_files: tuple[tuple[str, str], ...] + expected_returncode: int + expected_diagnostics: tuple[object, ...] + expected_descriptions: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class FormatDescriptionResolutionE2ECase: + description: str + extra_files: tuple[tuple[str, str], ...] + expected_returncode: int + expected_faults: tuple[tuple[str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index d0d28c6689..6c0a84d9c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2823,3 +2823,55 @@ def measure_union_fixture_compile( ) ) return min(measurements) + + +REQUIRED_DESCRIPTIONS_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_REQUIRED_DESCRIPTIONS_ORDERS_MODEL: str = ( + 'MODEL (description "One row per order");\n\nSELECT 1 AS order_id\n' +) +_REQUIRED_DESCRIPTIONS_COMPILE: tuple[str, ...] = ("--no-color", "compile", "--json", "--no-cache") + + +def hooked_model_file(*, model_name: str, hook: str) -> tuple[str, str]: + """Return a described model file whose post hook references ``hook``.""" + + return ( + f"models/{model_name}.sql", + f'MODEL (description "Orders with a hook", post_hooks [{hook}]);\n\nSELECT 1 AS order_id\n', + ) + + +def python_node_source( + *, + module: str, + decorator: str, + name: str, + docstring_line: str = "", + extra_import: str = "", + decorator_arguments: str = "", +) -> str: + """Render one decorated Python node module for required-description E2Es.""" + + return ( + f"from sqlbuild.{module} import {decorator}\n{extra_import}\n\n" + f"@{decorator}{decorator_arguments}\ndef {name}(ctx):\n{docstring_line} return None\n" + ) + + +def compile_inline_files( + *, tmp_path: Path, files: tuple[tuple[str, str], ...] +) -> subprocess.CompletedProcess[str]: + """Compile a described orders project extended with ``files``.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": REQUIRED_DESCRIPTIONS_PROJECT, + "models/orders.sql": _REQUIRED_DESCRIPTIONS_ORDERS_MODEL, + **dict(files), + }, + ) + return run_sqb(project_dir=project_dir, command=_REQUIRED_DESCRIPTIONS_COMPILE) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py new file mode 100644 index 0000000000..8c5edfcebc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -0,0 +1,669 @@ +"""E2E coverage of the P010 compile error for named resources without a description.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import asdict +from itertools import chain +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.compiler.compile.constants import DESCRIPTION_REQUIRED_INPUT_KINDS +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FormatDescriptionResolutionE2ECase, + PathDefaultDescriptionCase, + RequiredDescriptionAggregateCase, + RequiredDescriptionCase, + RequiredDescriptionPlanCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + REQUIRED_DESCRIPTIONS_PROJECT, + compile_inline_files, + hooked_model_file, + python_node_source, + require_sql_analysis_output, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_SEED_CSV: str = "product_type_id,label\n1,waffle\n" +_SEED_COLUMNS: str = ( + " columns:\n - name: product_type_id\n type: INTEGER\n" + " - name: label\n type: VARCHAR\n" +) +_ORDER_TOTALS_MODEL: tuple[str, str] = ( + "models/order_totals.sql", + 'MODEL (description "Order count");\n\nSELECT COUNT(*) AS order_count FROM __ref("orders")\n', +) +_SCENARIO_BODY: str = ( + "WITH\n__ref__orders AS (\n SELECT 1 AS order_id\n),\n" + "__expected__order_totals AS (\n SELECT 1 AS order_count\n)\nSELECT 1\n" +) + + +_PROVIDER_DESCRIBED: str = ( + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n" + ' """Client for the orders API."""\n\n' + ' base_url: str = "https://example.com"\n' +) +_PROVIDER_UNDESCRIBED: str = ( + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n" + ' base_url: str = "https://example.com"\n' +) + + +_CASES: tuple[RequiredDescriptionCase, ...] = ( + RequiredDescriptionCase( + description="model header", + missing_files=( + ("models/order_totals.sql", "MODEL (materialized table);\nSELECT 1 AS id\n"), + ), + described_files=( + ( + "models/order_totals.sql", + 'MODEL (description "Order totals", materialized table);\nSELECT 1 AS id\n', + ), + ), + expected_diagnostic=("P010", "models/order_totals.sql", 1), + expected_message="model 'order_totals' has no description", + expected_help_fragments=( + "to describe model 'order_totals', add this to the MODEL header in " + "models/order_totals.sql:\n" + " MODEL (\n" + ' description "What one row of order_totals represents",\n' + " ...\n" + " );", + ), + ), + RequiredDescriptionCase( + description="scenario header", + missing_files=( + ( + "tests/scenarios/orders__single_order.sql", + "SCENARIO ();\n\n" + _SCENARIO_BODY, + ), + _ORDER_TOTALS_MODEL, + ), + described_files=( + ( + "tests/scenarios/orders__single_order.sql", + 'SCENARIO (description "One order produces one row");\n\n' + _SCENARIO_BODY, + ), + _ORDER_TOTALS_MODEL, + ), + expected_diagnostic=("P010", "tests/scenarios/orders__single_order.sql", 1), + expected_message="scenario 'orders__single_order' has no description", + expected_help_fragments=( + "add this to the SCENARIO header in tests/scenarios/orders__single_order.sql:\n" + " SCENARIO (\n" + ' description "The behaviour orders__single_order proves",', + ), + ), + RequiredDescriptionCase( + description="seed YAML entry", + missing_files=( + ("seeds/product_types.csv", _SEED_CSV), + ("seeds/product_types.yml", "seeds:\n - name: product_types\n" + _SEED_COLUMNS), + ), + described_files=( + ("seeds/product_types.csv", _SEED_CSV), + ( + "seeds/product_types.yml", + "seeds:\n - name: product_types\n description: Product type catalogue\n" + + _SEED_COLUMNS, + ), + ), + expected_diagnostic=("P010", "seeds/product_types.yml", 2), + expected_message="seed 'product_types' has no description", + expected_help_fragments=( + "add `description` to its entry in seeds/product_types.yml:\n" + " - name: product_types\n" + " description: What one row of product_types represents", + ), + ), + RequiredDescriptionCase( + description="source YAML entry", + missing_files=( + ( + "sources/raw.yml", + "sources:\n - name: raw_customers\n schema: main\n table: customers\n", + ), + ), + described_files=( + ( + "sources/raw.yml", + "sources:\n - name: raw_customers\n description: Customer records\n" + " schema: main\n table: customers\n", + ), + ), + expected_diagnostic=("P010", "sources/raw.yml", 2), + expected_message="source 'raw_customers' has no description", + expected_help_fragments=( + "add `description` to its entry in sources/raw.yml:\n" + " - name: raw_customers\n", + ), + ), + RequiredDescriptionCase( + description="managed source described by its loader docstring", + missing_files=( + ("sources/raw.yml", "sources:\n - name: raw_customers\n managed: true\n"), + ( + "python/loaders/raw_customers.py", + python_node_source(module="loaders", decorator="loader", name="raw_customers"), + ), + ), + described_files=( + ("sources/raw.yml", "sources:\n - name: raw_customers\n managed: true\n"), + ( + "python/loaders/raw_customers.py", + python_node_source( + module="loaders", + decorator="loader", + name="raw_customers", + docstring_line=' """Customer records from the CRM export."""\n', + ), + ), + ), + expected_diagnostic=("P010", "sources/raw.yml", 2), + expected_message="source 'raw_customers' has no description", + expected_help_fragments=( + "or describe its loader 'raw_customers' in python/loaders/raw_customers.py with a " + 'docstring or @loader(description="What one row of raw_customers represents")', + ), + ), + RequiredDescriptionCase( + description="SQL function header", + missing_files=( + ( + "functions/sql/is_large_order.sql", + "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\n" + "amount > 100\n", + ), + ), + described_files=( + ( + "functions/sql/is_large_order.sql", + 'FUNCTION (\n description "Whether an order amount is large",\n' + " arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/sql/is_large_order.sql", 1), + expected_message="function 'is_large_order' has no description", + expected_help_fragments=( + "add this to the FUNCTION header in functions/sql/is_large_order.sql:\n" + " FUNCTION (\n" + ' description "What is_large_order returns",', + ), + ), + RequiredDescriptionCase( + description="table function header", + missing_files=( + ( + "functions/sql/order_lines.sql", + "FUNCTION (\n arguments (p_order_id INTEGER),\n" + " returns table (order_id INTEGER),\n);\n\nSELECT p_order_id AS order_id\n", + ), + ), + described_files=( + ( + "functions/sql/order_lines.sql", + 'FUNCTION (\n description "Lines of one order",\n' + " arguments (p_order_id INTEGER),\n" + " returns table (order_id INTEGER),\n);\n\nSELECT p_order_id AS order_id\n", + ), + ), + expected_diagnostic=("P010", "functions/sql/order_lines.sql", 1), + expected_message="function 'order_lines' has no description", + expected_help_fragments=('description "What order_lines returns"',), + ), + RequiredDescriptionCase( + description="Python UDF described by its docstring", + missing_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + described_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n" + ' """Whether an order amount is large."""\n' + " return amount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/python/is_large_order_py.py", 4), + expected_message="function 'is_large_order_py' has no description", + expected_help_fragments=( + "give its function in functions/python/is_large_order_py.py a docstring or add " + "this to @udf:\n" + ' @udf(description="What is_large_order_py returns", ...)', + ), + ), + RequiredDescriptionCase( + description="Python UDF described by its decorator", + missing_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + described_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + "@udf(\n" + ' description="Whether an order amount is large",\n' + ' arguments={"amount": "INTEGER"},\n' + ' returns="BOOLEAN",\n' + ' runtime_version="3.11",\n' + ")\n" + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/python/is_large_order_py.py", 4), + expected_message="function 'is_large_order_py' has no description", + expected_help_fragments=(), + ), + RequiredDescriptionCase( + description="named SQL hook header", + missing_files=( + ("hooks/sql/analyze_orders.sql", "HOOK ();\n\nSELECT 1\n"), + hooked_model_file(model_name="analyzed_orders", hook='sql("analyze_orders")'), + ), + described_files=( + ( + "hooks/sql/analyze_orders.sql", + 'HOOK (description "What analyze_orders does");\n\nSELECT 1\n', + ), + hooked_model_file(model_name="analyzed_orders", hook='sql("analyze_orders")'), + ), + expected_diagnostic=("P010", "hooks/sql/analyze_orders.sql", 1), + expected_message="hook 'analyze_orders' has no description", + expected_help_fragments=( + "add this to the HOOK header in hooks/sql/analyze_orders.sql:\n" + ' HOOK (description "What analyze_orders does");', + ), + ), + RequiredDescriptionCase( + description="Python hook", + missing_files=( + ( + "hooks/python/notify_orders.py", + python_node_source(module="hooks", decorator="hook", name="notify_orders"), + ), + hooked_model_file(model_name="notified_orders", hook='python("notify_orders")'), + ), + described_files=( + ( + "hooks/python/notify_orders.py", + python_node_source( + module="hooks", + decorator="hook", + name="notify_orders", + docstring_line=' """Notify the orders channel."""\n', + ), + ), + hooked_model_file(model_name="notified_orders", hook='python("notify_orders")'), + ), + expected_diagnostic=("P010", "hooks/python/notify_orders.py", 4), + expected_message="hook 'notify_orders' has no description", + expected_help_fragments=( + "give its function in hooks/python/notify_orders.py a docstring or pass a " + "description to @hook:\n" + ' @hook(description="What notify_orders does", ...)', + ), + ), + RequiredDescriptionCase( + description="loader without a source declaration", + missing_files=( + ( + "python/loaders/raw_returns.py", + python_node_source(module="loaders", decorator="loader", name="raw_returns"), + ), + ), + described_files=( + ( + "python/loaders/raw_returns.py", + python_node_source( + module="loaders", + decorator="loader", + name="raw_returns", + docstring_line=' """Returned orders from the support desk."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/loaders/raw_returns.py", 4), + expected_message="loader 'raw_returns' has no description", + expected_help_fragments=('@loader(description="What raw_returns does", ...)',), + ), + RequiredDescriptionCase( + description="task", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source(module="tasks", decorator="task", name="refresh_exports"), + ), + ), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=' """Refresh the order exports."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' has no description", + expected_help_fragments=('@task(description="What refresh_exports does", ...)',), + ), + RequiredDescriptionCase( + description="asset", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source(module="assets", decorator="asset", name="order_report"), + ), + ), + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + docstring_line=' """Weekly order report file."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' has no description", + expected_help_fragments=('@asset(description="What order_report does", ...)',), + ), + RequiredDescriptionCase( + description="check", + missing_files=( + ( + "python/checks/orders_present.py", + python_node_source( + module="checks", + decorator="check", + name="orders_present", + extra_import="from sqlbuild.refs import model\n", + decorator_arguments='(depends_on=model("orders"))', + ), + ), + ), + described_files=( + ( + "python/checks/orders_present.py", + python_node_source( + module="checks", + decorator="check", + name="orders_present", + extra_import="from sqlbuild.refs import model\n", + decorator_arguments='(depends_on=model("orders"))', + docstring_line=' """Orders exist after the build."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/checks/orders_present.py", 5), + expected_message="check 'orders_present' has no description", + expected_help_fragments=('@check(description="What orders_present does", ...)',), + ), + RequiredDescriptionCase( + description="provider", + missing_files=(("providers/orders_api.py", _PROVIDER_UNDESCRIBED),), + described_files=(("providers/orders_api.py", _PROVIDER_DESCRIBED),), + expected_diagnostic=("P010", "providers/orders_api.py", 4), + expected_message="provider 'orders_api' has no description", + expected_help_fragments=( + "add a class docstring in providers/orders_api.py:\n" + " class OrdersApi(Provider):\n" + ' """What orders_api provides."""', + ), + ), +) + + +_PATH_DEFAULT_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n\n' + '[path_defaults.marts]\ndescription = "Mart model"\n' +) +_OWN_DESCRIPTION_MODEL: str = 'MODEL (description "Own description");\n\nSELECT 1 AS id\n' +_INHERITING_MODEL: str = "MODEL ();\n\nSELECT 1 AS id\n" + + +_TYPED_NULL_FIXTURE_FILES: tuple[tuple[str, str], ...] = ( + ("models/up.sql", 'MODEL (description "Upstream rows");\n\nSELECT 1 AS a\n'), + ( + "models/marts/a_model.sql", + 'MODEL (description "Own description");\n\nSELECT a\nFROM __ref("up")\n', + ), + ("models/marts/b_model.sql", _INHERITING_MODEL), + ( + "tests/unit/test_a_model.sql", + 'TEST (description "Null values pass through");\n\n' + "WITH __ref__up AS (\n SELECT CAST(NULL AS INTEGER) AS a\n),\n\n" + "__expected__a_model AS (\n SELECT CAST(NULL AS INTEGER) AS a\n)\n\nSELECT 1\n", + ), +) + + +_ALL_UNDESCRIBED_FILES: tuple[tuple[str, str], ...] = ( + *dict(chain.from_iterable(case.missing_files for case in _CASES)).items(), + ("models/undescribed_orders.sql", "MODEL (materialized view);\n\nSELECT 1 AS id\n"), +) + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionCase(**asdict(case)) for case in _CASES], + ids=lambda case: case.description, +) +def test_given_resource_without_description_when_compiling_then_reports_p010_with_snippet( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.missing_files + ) + + diagnostics, text = require_sql_analysis_output(result) + payload: dict[str, Any] = json.loads(result.stdout) + assert result.returncode == 1, result.stdout + result.stderr + assert diagnostics == (test_case.expected_diagnostic,) + assert payload["diagnostics"][0]["message"] == test_case.expected_message + assert all(fragment in text for fragment in test_case.expected_help_fragments), text + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionCase(**asdict(case)) for case in _CASES], + ids=lambda case: case.description, +) +def test_given_described_resource_when_compiling_then_compiles_cleanly( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.described_files + ) + + diagnostics, text = require_sql_analysis_output(result) + assert result.returncode == 0, result.stdout + result.stderr + assert diagnostics == () + assert test_case.expected_message not in text + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionAggregateCase( + description="every required kind undescribed in one project", + files=_ALL_UNDESCRIBED_FILES, + expected_codes=frozenset({"P010"}), + expected_kinds=frozenset(DESCRIPTION_REQUIRED_INPUT_KINDS.values()), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_resource_kind_undescribed_when_compiling_then_one_run_reports_each( + test_case: RequiredDescriptionAggregateCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.files + ) + + payload: dict[str, Any] = json.loads(result.stdout) + codes: set[str] = {item["code"] for item in payload["diagnostics"]} + kinds: set[str] = {item["message"].split(" ", 1)[0] for item in payload["diagnostics"]} + assert result.returncode == 1, result.stdout + result.stderr + assert codes == test_case.expected_codes + assert kinds == test_case.expected_kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionPlanCase( + description="undescribed model blocks plan", + model_sql="MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + expected_returncode=1, + expected_stderr_fragment=( + "[P010] models/orders.sql:1:1: model 'orders' has no description" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_undescribed_model_when_planning_then_plan_is_blocked( + test_case: RequiredDescriptionPlanCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": REQUIRED_DESCRIPTIONS_PROJECT, + "models/orders.sql": test_case.model_sql, + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "plan") + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_stderr_fragment in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + PathDefaultDescriptionCase( + description="described model sorts before the inheriting model", + model_files=( + ("models/marts/a_model.sql", _OWN_DESCRIPTION_MODEL), + ("models/marts/b_model.sql", _INHERITING_MODEL), + ), + expected_returncode=0, + expected_diagnostics=(), + expected_descriptions=(("a_model", "Own description"), ("b_model", "Mart model")), + ), + PathDefaultDescriptionCase( + description="inheriting model sorts before the described model", + model_files=( + ("models/marts/a_model.sql", _INHERITING_MODEL), + ("models/marts/b_model.sql", _OWN_DESCRIPTION_MODEL), + ), + expected_returncode=0, + expected_diagnostics=(), + expected_descriptions=(("a_model", "Mart model"), ("b_model", "Own description")), + ), + ], + ids=lambda case: case.description, +) +def test_given_path_default_description_when_compiling_then_every_model_keeps_its_description( + test_case: PathDefaultDescriptionCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={"sqlbuild_project.toml": _PATH_DEFAULT_PROJECT, **dict(test_case.model_files)}, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + dag: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "dag", "--json") + ) + + nodes: list[dict[str, Any]] = json.loads(dag.stdout)["nodes"] + assert compiled.returncode == test_case.expected_returncode, compiled.stdout + compiled.stderr + assert tuple(json.loads(compiled.stdout)["diagnostics"]) == test_case.expected_diagnostics + assert dag.returncode == test_case.expected_returncode, dag.stdout + dag.stderr + assert sorted((node["name"], node["description"]) for node in nodes) == list( + test_case.expected_descriptions + ) + + +@pytest.mark.parametrize( + "test_case", + [ + FormatDescriptionResolutionE2ECase( + description="typed-null fixture tests do not hide a path-default description", + extra_files=(), + expected_returncode=0, + expected_faults=(), + ), + FormatDescriptionResolutionE2ECase( + description="a genuinely missing description is still reported", + extra_files=(("models/bare.sql", "MODEL (materialized table);\n\nSELECT 1 AS id\n"),), + expected_returncode=1, + expected_faults=(("models/bare.sql", "description-present"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_inherited_descriptions_when_format_checking_then_faults_match_compile( + test_case: FormatDescriptionResolutionE2ECase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _PATH_DEFAULT_PROJECT, + **dict(_TYPED_NULL_FIXTURE_FILES), + **dict(test_case.extra_files), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "format", "--check", "--json") + ) + + violations: list[dict[str, Any]] = json.loads(result.stdout)["violations"] + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert ( + tuple( + (Path(item["file"]).relative_to(project_dir).as_posix(), item["code"]) + for item in violations + ) + == test_case.expected_faults + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 2f84b7dea5..c08a3bdda2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -200,3 +200,4 @@ class RulesCacheEditCase: description: str edit: Callable[[Path], None] expected_exit_code: int = 1 + expected_diagnostics_fragment: str = "" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 831de8a32c..8e722ff862 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -9,6 +9,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) @@ -371,3 +372,20 @@ def rule_cache_counts(project_dir: Path) -> tuple[int, int]: result: subprocess.CompletedProcess[str] = run_compile_cli(project_dir) timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] return timings["rule_cache_hits"], timings["rule_cache_misses"] + + +def move_order_totals_description_to_path_defaults(root: Path) -> None: + """Drop the mart model's header description and supply it from `[path_defaults]`.""" + + replace_project_text( + root, + "models/marts/order_totals.sql", + "MODEL (description 'Test model order_totals.',\n", + "MODEL (\n", + ) + config: Path = root / "sqlbuild_project.toml" + config.write_text( + config.read_text(encoding="utf-8") + + '\n[path_defaults.marts]\ndescription = "Order totals per order"\n', + encoding="utf-8", + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py index 96611b12f6..a66ace2c1b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py @@ -12,6 +12,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RulesCacheEditCase from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( INCREMENTAL_RULES_PROJECT, + move_order_totals_description_to_path_defaults, rule_cache_counts, rules_compile_outcome, write_project_files, @@ -36,6 +37,29 @@ root, "models/staging/stg_orders.sql", " contract enforced,\n", "" ), ), + RulesCacheEditCase( + "model_description_edited", + lambda root: replace_project_text( + root, + "models/staging/stg_orders.sql", + "Test model stg_orders.", + "Staged order rows.", + ), + ), + RulesCacheEditCase( + "model_description_removed", + lambda root: replace_project_text( + root, + "models/marts/order_totals.sql", + "MODEL (description 'Test model order_totals.',\n", + "MODEL (\n", + ), + expected_diagnostics_fragment='"code": "P010"', + ), + RulesCacheEditCase( + "model_description_moved_to_path_defaults", + move_order_totals_description_to_path_defaults, + ), RulesCacheEditCase( "model_added", lambda root: write_project_file( @@ -131,6 +155,7 @@ def test_given_warm_rules_cache_when_one_input_changes_then_compile_equals_uncac rewarmed: tuple[int, str, str] = rules_compile_outcome(project_dir) assert edited[0] == test_case.expected_exit_code + assert test_case.expected_diagnostics_fragment in edited[1] assert edited == oracle assert rewarmed == oracle assert rule_cache_counts(project_dir)[1] == 0 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 5abecd9b99..9287a1e455 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -357,13 +357,15 @@ class FormatScopeIntegrationTestCase: @dataclass(frozen=True) -class FormatWarningIntegrationTestCase: - """One non-failing format warning expectation.""" +class FormatDescriptionFaultIntegrationTestCase: + """A missing description faults format check and write mode alike.""" description: str + authored_sql: str expected_exit_code: int expected_code: str expected_severity: str + expected_formatted_sql: str @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index 531b135075..af96e4f2dd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -16,11 +16,11 @@ DescriptionFormatIntegrationTestCase, DollarQuoteFormatIntegrationTestCase, FormatCompileIntegrationTestCase, + FormatDescriptionFaultIntegrationTestCase, FormatPathArgumentsIntegrationTestCase, FormatSafetyIntegrationTestCase, FormatScopeIntegrationTestCase, FormatterDeclineIntegrationTestCase, - FormatWarningIntegrationTestCase, FromValuesFormatIntegrationTestCase, LayoutOnlyFormatIntegrationTestCase, LeadingCteCommentFormatIntegrationTestCase, @@ -871,17 +871,19 @@ def test_given_format_selectors_when_scoping_then_paths_and_default_exclusions_a @pytest.mark.parametrize( "test_case", [ - FormatWarningIntegrationTestCase( - description="missing model description warns without failing format", - expected_exit_code=0, + FormatDescriptionFaultIntegrationTestCase( + description="missing model description fails check and write modes", + authored_sql="MODEL (materialized table);\nselect 1 as order_id\n", + expected_exit_code=1, expected_code="description-present", - expected_severity="warning", + expected_severity="fault", + expected_formatted_sql="MODEL (materialized table);\nSELECT 1 AS order_id\n", ) ], ids=lambda case: case.description, ) -def test_given_missing_description_when_formatting_then_warning_does_not_fail( - test_case: FormatWarningIntegrationTestCase, +def test_given_missing_description_when_formatting_then_check_and_write_fault( + test_case: FormatDescriptionFaultIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -890,26 +892,29 @@ def test_given_missing_description_when_formatting_then_warning_does_not_fail( ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL (materialized table);\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text(test_case.authored_sql, encoding="utf-8") - write_exit: int = main(["--project-dir", str(tmp_path), "format"]) - _ = capsys.readouterr() text_exit: int = main(["--project-dir", str(tmp_path), "format", "--check"]) text_output: str = capsys.readouterr().out json_exit: int = main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) payload: dict[str, object] = json.loads(capsys.readouterr().out) - assert write_exit == test_case.expected_exit_code assert text_exit == test_case.expected_exit_code - assert f"warning[{test_case.expected_code}]" in text_output + assert f"error[{test_case.expected_code}]" in text_output assert json_exit == test_case.expected_exit_code - assert payload["faults"] == 0 - assert payload["warnings"] == 1 + assert payload["faults"] == 1 + assert payload["warnings"] == 0 violations: object = payload["violations"] assert isinstance(violations, list) assert violations[0]["code"] == test_case.expected_code assert violations[0]["severity"] == test_case.expected_severity + write_exit: int = main(["--project-dir", str(tmp_path), "format"]) + write_output: str = capsys.readouterr().out + assert write_exit == test_case.expected_exit_code + assert f"error[{test_case.expected_code}]" in write_output + assert model.read_text(encoding="utf-8") == test_case.expected_formatted_sql + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index cae1745351..28b16aacf7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -21,6 +21,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import RequiredDescriptionInputs @dataclass(frozen=True) @@ -1099,3 +1100,32 @@ class CompactLineageCacheEncodingTestCase: output_column: str resource_names: dict[int, str] expected_lineage_columns: int = 2 + + +@dataclass(frozen=True) +class RequiredDescriptionUnitCase: + """One resource kind checked by the required-description compile rule.""" + + description: str + build: Callable[[str | None], RequiredDescriptionInputs] + expected_message: str + expected_location: tuple[str, int, int] + + +@dataclass(frozen=True) +class DescriptionInputClassificationCase: + """Discovered input fields split between required and exempt description kinds.""" + + description: str + required: frozenset[str] + exempt: frozenset[str] + expected_fields: frozenset[str] + + +@dataclass(frozen=True) +class RequiredDescriptionLocationCase: + """Where a P010 finding points when comments or docstrings precede the declaration.""" + + description: str + build: Callable[[], RequiredDescriptionInputs] + expected_location: tuple[str, int, int] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py index 03f6de49dd..a88b55a540 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py @@ -45,7 +45,7 @@ def test_given_static_column_contracts_when_attaching_then_reuses_only_base_conf """.strip() + "\n", "models/customers.sql": """ -MODEL ( +MODEL (description "Test model customers.", columns (customer_id (type INTEGER, nullable false, description "Customer id")), ); @@ -53,7 +53,7 @@ def test_given_static_column_contracts_when_attaching_then_reuses_only_base_conf """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns (order_id (type BIGINT, nullable true, description "Order id")), ); @@ -118,7 +118,7 @@ def test_given_templated_columns_when_attaching_then_resolves_each_model_context """.strip() + "\n", "models/customers.sql": """ -MODEL ( +MODEL (description "Test model customers.", columns ( customer_id ( type ${identifier_type}, @@ -131,7 +131,7 @@ def test_given_templated_columns_when_attaching_then_resolves_each_model_context """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id ( type ${identifier_type}, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 52e442f5dd..d55f1ec672 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,6 +1,6 @@ import sqlite3 from collections.abc import Callable -from dataclasses import replace +from dataclasses import dataclass, replace from pathlib import Path from types import MappingProxyType from typing import cast @@ -15,6 +15,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, ) +from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( + missing_description_diagnostics, +) from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros, load_project_macros, @@ -34,8 +37,12 @@ CompiledProject, CompiledRelationLocation, CompiledSqlTest, + CompileModelInput, CompileProjectInputs, CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlFunctionInput, DeclarationExpansionContext, DeclarationResolutionContext, DeclarationRuntimeProjection, @@ -43,9 +50,22 @@ LoadedMacro, MacroContext, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import ( + DiscoveredLoaderFunction, + DiscoveredMacroFile, + DiscoveredProjectInputs, + DiscoveredProvider, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, + DiscoveredTaskFunction, +) from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -63,6 +83,14 @@ VisibilityRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind +from sqlbuild.providers import Provider +from sqlbuild.spec.contracts.models import ( + LocalConfig, + ProjectConfig, + SchemaModelEntry, + SchemaSeedEntry, + SourceEntry, +) from sqlbuild.sql_values.types import CollectionRendering DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( @@ -485,3 +513,251 @@ def stored_analysis_contents(*, context: AnalysisCacheContext, cache_key: str) - "SELECT payload FROM model_analysis WHERE cache_key = ?", (cache_key,) ).fetchone() return row[0] + + +REQUIRED_DESCRIPTION_BASE_INPUTS: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), +) + + +@dataclass(frozen=True) +class RequiredDescriptionInputs: + """Compile inputs passed to the required-description check.""" + + discovered_inputs: DiscoveredProjectInputs = REQUIRED_DESCRIPTION_BASE_INPUTS + model_inputs: tuple[CompileModelInput, ...] = () + seed_inputs: tuple[CompileSeedInput, ...] = () + source_inputs: tuple[CompileSourceInput, ...] = () + function_inputs: tuple[CompileSqlFunctionInput, ...] = () + + +class OrdersApi(Provider): + """Client for the orders API.""" + + +class UndescribedApi(Provider): + pass + + +_PROVIDER_CLASS_BY_UNDESCRIBED: dict[bool, type[Provider]] = { + False: OrdersApi, + True: UndescribedApi, +} + + +def refresh_exports_task(ctx: object) -> None: + del ctx + + +def documented_returns_loader(ctx: object) -> None: + """Returned orders from the support desk.""" + + del ctx + + +def model_description_inputs( + description: str | None, + *, + contents: str = "-- totals\nMODEL (materialized table);\nSELECT 1 AS id\n", +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + model_inputs=( + CompileModelInput( + model_file=DiscoveredSqlModelFile( + file_path=Path("/project/models/order_totals.sql"), + relative_path=Path("models/order_totals.sql"), + contents=contents, + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="SELECT 1 AS id", + ), + schema_entry=SchemaModelEntry(name="order_totals", description=description), + ), + ) + ) + + +def scenario_description_inputs( + description: str | None, *, contents: str = "\nSCENARIO ();\nSELECT 1\n" +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + scenario_files=( + DiscoveredSqlScenarioFile( + file_path=Path("/project/tests/scenarios/orders__paid.sql"), + relative_path=Path("tests/scenarios/orders__paid.sql"), + contents=contents, + header_values={"description": description}, + sql_body="SELECT 1", + name="orders__paid", + ), + ), + ) + ) + + +def seed_description_inputs(description: str | None) -> RequiredDescriptionInputs: + contents: str = "seeds:\n - name: product_types\n" + return RequiredDescriptionInputs( + seed_inputs=( + CompileSeedInput( + seed_file=DiscoveredSeedFile( + file_path=Path("/project/seeds/product_types.csv"), + relative_path=Path("seeds/product_types.csv"), + ), + schema_entry=SchemaSeedEntry(name="product_types", description=description), + schema_file=DiscoveredSchemaFile( + file_path=Path("/project/seeds/product_types.yml"), + relative_path=Path("seeds/product_types.yml"), + contents=contents, + model_entries=(), + seed_entries=(), + ), + ), + ) + ) + + +def returns_source_input(*, description: str | None, loader: str | None) -> CompileSourceInput: + return CompileSourceInput( + source_entry=SourceEntry(name="raw_returns", description=description, loader=loader), + source_file=DiscoveredSourceFile( + file_path=Path("/project/sources/raw.yml"), + relative_path=Path("sources/raw.yml"), + contents="sources:\n - name: raw_orders\n description: Orders\n" + " - name: raw_returns\n", + source_entries=(), + ), + ) + + +def source_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + source_inputs=(returns_source_input(description=description, loader=None),) + ) + + +def returns_loader(description: str | None) -> DiscoveredLoaderFunction: + return DiscoveredLoaderFunction( + file_path=Path("/project/python/loaders/raw_returns.py"), + relative_path=Path("python/loaders/raw_returns.py"), + name="raw_returns", + function=documented_returns_loader, + description=description, + ) + + +def loader_backed_source_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + source_inputs=(returns_source_input(description=None, loader="raw_returns"),), + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) + ), + ) + + +def loader_only_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) + ) + ) + + +def function_description_inputs( + description: str | None, + *, + contents: str = "FUNCTION (returns BOOLEAN);\n\nTRUE\n", + relative_path: Path = Path("functions/sql/is_large_order.sql"), + language: FunctionLanguage = FunctionLanguage.SQL, +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + function_inputs=( + CompileSqlFunctionInput( + function_file=DiscoveredSqlFunctionFile( + file_path=Path("/project") / relative_path, + relative_path=relative_path, + contents=contents, + header_values={}, + body_sql="TRUE", + ), + name="is_large_order", + arguments=(), + returns="BOOLEAN", + body_sql="TRUE", + description=description, + language=language, + ), + ) + ) + + +def sql_hook_description_inputs( + description: str | None, *, contents: str = "HOOK ();\nSELECT 1\n" +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + sql_hook_files=( + DiscoveredSqlHookFile( + file_path=Path("/project/hooks/sql/analyze_orders.sql"), + relative_path=Path("hooks/sql/analyze_orders.sql"), + contents=contents, + header_values={}, + sql_body="SELECT 1", + name="analyze_orders", + description=description, + ), + ), + ) + ) + + +def task_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + task_functions=( + DiscoveredTaskFunction( + file_path=Path("/project/python/tasks/refresh_exports.py"), + relative_path=Path("python/tasks/refresh_exports.py"), + name="refresh_exports", + function=refresh_exports_task, + description=description, + ), + ), + ) + ) + + +def provider_description_inputs(description: str | None) -> RequiredDescriptionInputs: + provider_class: type[Provider] = _PROVIDER_CLASS_BY_UNDESCRIBED[description is None] + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + providers=( + DiscoveredProvider( + file_path=Path("/project/providers/orders_api.py"), + relative_path=Path("providers/orders_api.py"), + name="orders_api", + provider_class=provider_class, + settings=provider_class(), + ), + ), + ) + ) + + +def required_description_diagnostics( + inputs: RequiredDescriptionInputs, +) -> tuple[CompilerDiagnostic, ...]: + return missing_description_diagnostics( + discovered_inputs=inputs.discovered_inputs, + model_inputs=inputs.model_inputs, + seed_inputs=inputs.seed_inputs, + source_inputs=inputs.source_inputs, + function_inputs=inputs.function_inputs, + ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py new file mode 100644 index 0000000000..79ecef24b8 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -0,0 +1,229 @@ +"""Unit tests for the required-description compile check (P010).""" + +from __future__ import annotations + +import inspect +from dataclasses import asdict, fields +from functools import partial +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.constants import ( + DESCRIPTION_EXEMPT_INPUTS, + DESCRIPTION_REQUIRED_INPUT_KINDS, +) +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import FunctionLanguage +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DescriptionInputClassificationCase, + RequiredDescriptionLocationCase, + RequiredDescriptionUnitCase, +) +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + UndescribedApi, + documented_returns_loader, + function_description_inputs, + loader_backed_source_description_inputs, + loader_only_description_inputs, + model_description_inputs, + provider_description_inputs, + refresh_exports_task, + required_description_diagnostics, + scenario_description_inputs, + seed_description_inputs, + source_description_inputs, + sql_hook_description_inputs, + task_description_inputs, +) + +_UDF_AFTER_MENTIONS: str = ( + '"""Mentions @udf before the decorator."""\n' + "# @udf in a comment\n" + "from sqlbuild.functions import udf\n\n\n" + '@udf(returns="BOOLEAN")\n' + "def main(amount: int) -> bool:\n" + " return amount > 100\n" +) + + +CASES: tuple[RequiredDescriptionUnitCase, ...] = ( + RequiredDescriptionUnitCase( + "model header after a leading comment", + model_description_inputs, + "model 'order_totals' has no description", + ("models/order_totals.sql", 2, 1), + ), + RequiredDescriptionUnitCase( + "scenario header", + scenario_description_inputs, + "scenario 'orders__paid' has no description", + ("tests/scenarios/orders__paid.sql", 2, 1), + ), + RequiredDescriptionUnitCase( + "seed YAML entry", + seed_description_inputs, + "seed 'product_types' has no description", + ("seeds/product_types.yml", 2, 1), + ), + RequiredDescriptionUnitCase( + "source YAML entry among others", + source_description_inputs, + "source 'raw_returns' has no description", + ("sources/raw.yml", 4, 1), + ), + RequiredDescriptionUnitCase( + "source described through its loader", + loader_backed_source_description_inputs, + "source 'raw_returns' has no description", + ("sources/raw.yml", 4, 1), + ), + RequiredDescriptionUnitCase( + "loader without a source declaration", + loader_only_description_inputs, + "loader 'raw_returns' has no description", + ("python/loaders/raw_returns.py", documented_returns_loader.__code__.co_firstlineno, 1), + ), + RequiredDescriptionUnitCase( + "SQL function header", + function_description_inputs, + "function 'is_large_order' has no description", + ("functions/sql/is_large_order.sql", 1, 1), + ), + RequiredDescriptionUnitCase( + "named SQL hook", + sql_hook_description_inputs, + "hook 'analyze_orders' has no description", + ("hooks/sql/analyze_orders.sql", 1, 1), + ), + RequiredDescriptionUnitCase( + "task", + task_description_inputs, + "task 'refresh_exports' has no description", + ("python/tasks/refresh_exports.py", refresh_exports_task.__code__.co_firstlineno, 1), + ), + RequiredDescriptionUnitCase( + "provider", + provider_description_inputs, + "provider 'orders_api' has no description", + ("providers/orders_api.py", inspect.getsourcelines(UndescribedApi)[1], 1), + ), +) + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], + ids=lambda case: case.description, +) +def test_given_undescribed_resource_when_checking_then_reports_one_p010_at_declaration( + test_case: RequiredDescriptionUnitCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build(None) + ) + + assert [ + (item.code, item.message, (str(item.path), item.line, item.column)) for item in diagnostics + ] == [("P010", test_case.expected_message, test_case.expected_location)] + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], + ids=lambda case: case.description, +) +def test_given_described_resource_when_checking_then_reports_nothing( + test_case: RequiredDescriptionUnitCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build("Described resource.") + ) + + assert test_case.expected_message not in [item.message for item in diagnostics] + assert diagnostics == () + + +@pytest.mark.parametrize( + "test_case", + [ + DescriptionInputClassificationCase( + description="every discovered input field is required or exempt", + required=frozenset(DESCRIPTION_REQUIRED_INPUT_KINDS), + exempt=frozenset(DESCRIPTION_EXEMPT_INPUTS), + expected_fields=frozenset(item.name for item in fields(DiscoveredProjectInputs)), + ), + ], + ids=lambda case: case.description, +) +def test_given_discovered_input_fields_when_classifying_then_every_field_is_checked_or_exempt( + test_case: DescriptionInputClassificationCase, +) -> None: + assert not test_case.required & test_case.exempt + assert test_case.required | test_case.exempt == test_case.expected_fields + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionLocationCase( + description="MODEL header after a block comment that mentions MODEL (", + build=partial( + model_description_inputs, + None, + contents="/*\nMODEL (old)\n*/\n-- note\nMODEL (materialized table);\nSELECT 1\n", + ), + expected_location=("models/order_totals.sql", 5, 1), + ), + RequiredDescriptionLocationCase( + description="FUNCTION header after a leading comment", + build=partial( + function_description_inputs, + None, + contents="-- returns a flag\nFUNCTION (returns BOOLEAN);\n\nTRUE\n", + ), + expected_location=("functions/sql/is_large_order.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="HOOK header after a leading block comment", + build=partial( + sql_hook_description_inputs, None, contents="/* analyze */\nHOOK ();\nSELECT 1\n" + ), + expected_location=("hooks/sql/analyze_orders.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="SCENARIO header after a leading comment", + build=partial( + scenario_description_inputs, None, contents="-- paid\nSCENARIO ();\nSELECT 1\n" + ), + expected_location=("tests/scenarios/orders__paid.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="@udf decorator after docstring and comment mentions", + build=partial( + function_description_inputs, + None, + contents=_UDF_AFTER_MENTIONS, + relative_path=Path("functions/python/is_large_order.py"), + language=FunctionLanguage.PYTHON, + ), + expected_location=("functions/python/is_large_order.py", 6, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_comments_before_declaration_when_checking_then_points_at_declaration( + test_case: RequiredDescriptionLocationCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build() + ) + + assert [(str(item.path), item.line, item.column) for item in diagnostics] == [ + test_case.expected_location + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index b8a1d47041..0425fc326c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -620,6 +620,7 @@ class ParseSourcesYamlDltTestCase: expected_dlt_names: tuple[str, ...] expected_schemas: tuple[str | None, ...] expected_destination_config: dict[str, object] = field(default_factory=dict) + expected_descriptions: tuple[str | None, ...] = () @dataclass(frozen=True) @@ -760,8 +761,9 @@ class DiscoverLoaderFunctionsTestCase: description: str files: dict[str, str] expected_names: tuple[str, ...] - expected_targets: tuple[str | None, ...] - expected_dependency_counts: tuple[int, ...] + expected_targets: tuple[str | None, ...] = () + expected_dependency_counts: tuple[int, ...] = () + expected_descriptions: tuple[str | None, ...] = () expected_write_strategies: tuple[str | None, ...] = () expected_cursor_columns: tuple[str | None, ...] = () expected_unique_keys: tuple[tuple[str, ...], ...] = () @@ -928,6 +930,7 @@ class FunctionHeaderKeysTestCase: contents: str expected_error: str = "" expected_keys: tuple[str, ...] = () + expected_description: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index 8acb9febbe..560c3393e6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -227,3 +227,53 @@ def test_given_loader_import_error_when_discovering_loaders_then_raises_clear_er with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): discover_python_node_functions(project_dir=tmp_path) + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoverLoaderFunctionsTestCase( + description="loader descriptions come from the decorator or the docstring", + files={ + "python/loaders/orders.py": ''' +from sqlbuild.loaders import loader + +@loader +def raw_orders(ctx): + """Orders from the storefront export.""" + return [] + +@loader(description="Refunds from the payments export") +def raw_refunds(ctx): + """Ignored because the decorator wins.""" + return [] + +@loader +def raw_returns(ctx): + return [] +''', + }, + expected_names=("raw_orders", "raw_refunds", "raw_returns"), + expected_descriptions=( + "Orders from the storefront export.", + "Refunds from the payments export", + None, + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_loader_descriptions_when_discovering_then_decorator_wins_over_docstring( + test_case: DiscoverLoaderFunctionsTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + file_path: Path = tmp_path / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(contents, encoding="utf-8") + + result: tuple[DiscoveredLoaderFunction, ...] = tuple( + sorted(discover_python_node_functions(project_dir=tmp_path).loaders, key=lambda x: x.name) + ) + + assert tuple(loader.name for loader in result) == test_case.expected_names + assert tuple(loader.description for loader in result) == test_case.expected_descriptions diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py index 2c9d80068c..d6b6e26573 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py @@ -21,10 +21,11 @@ description="sql udf with every supported key", file_name="normalize_status.sql", contents=( - "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + 'FUNCTION (\n description "Normalise an order status",\n' + " arguments (status VARCHAR),\n returns VARCHAR,\n" " database analytics,\n schema udfs,\n tags [orders],\n);\n\nlower(status)\n" ), - expected_keys=("arguments", "database", "returns", "schema", "tags"), + expected_keys=("arguments", "database", "description", "returns", "schema", "tags"), ), FunctionHeaderKeysTestCase( description="table function with a returns table declaration", @@ -68,11 +69,10 @@ def test_given_supported_function_header_keys_when_parsing_then_accepts_header( file_name="expand_order.sql", contents=( 'FUNCTION (\n description "Expand one order",\n arguments (order_id INTEGER),\n' - " returns table (order_id INTEGER),\n owner orders_team,\n);\n\nSELECT order_id\n" - ), - expected_error=( - "FUNCTION() in 'expand_order.sql:2' has unsupported keys: description, owner" + " returns table (order_id INTEGER),\n owner orders_team,\n team sales,\n);\n\n" + "SELECT order_id\n" ), + expected_error=("FUNCTION() in 'expand_order.sql:5' has unsupported keys: owner, team"), ), ], ids=lambda case: case.description, @@ -114,5 +114,55 @@ def test_given_unknown_python_udf_key_when_parsing_then_rejects_key_with_locatio assert raised.value.message == test_case.expected_error +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeysTestCase( + description="docstring describes the udf", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN")\n' + 'def main(amount):\n """Whether an order amount is large."""\n' + " return amount > 100\n" + ), + expected_description="Whether an order amount is large.", + ), + FunctionHeaderKeysTestCase( + description="decorator description wins over the docstring", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(description="Large order flag", arguments={"amount": "INTEGER"}, ' + 'returns="BOOLEAN")\n' + 'def main(amount):\n """Whether an order amount is large."""\n' + " return amount > 100\n" + ), + expected_description="Large order flag", + ), + FunctionHeaderKeysTestCase( + description="undocumented udf has no description", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN")\n' + "def main(amount):\n return amount > 100\n" + ), + expected_description=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_udf_when_parsing_then_description_comes_from_decorator_or_docstring( + test_case: FunctionHeaderKeysTestCase, +) -> None: + header_values: dict[str, object] + header_values, _, _ = parse_python_function( + contents=test_case.contents, file_path=Path(test_case.file_name) + ) + + assert header_values.get("description") == test_case.expected_description + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 8ff38e7023..e68bbc6dc7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -483,6 +483,7 @@ def test_given_ingestr_source_yaml_when_parsing_then_stores_typed_integration_co schema: main resources: - name: raw_orders + description: Orders from the storefront database table: orders write_disposition: merge primary_key: order_id @@ -497,6 +498,7 @@ def test_given_ingestr_source_yaml_when_parsing_then_stores_typed_integration_co expected_dlt_names=("orders", "customers"), expected_schemas=("raw", "raw_customers_override"), expected_destination_config={"naming_convention": "sql_cs_v1", "create_indexes": True}, + expected_descriptions=("Orders from the storefront database", None), ) ], ids=lambda case: case.description, @@ -526,6 +528,8 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( tuple(config.resource.dlt_name for config in typed_configs) == test_case.expected_dlt_names ) assert tuple(entry.schema for entry in source_entries) == test_case.expected_schemas + assert tuple(entry.description for entry in source_entries) == test_case.expected_descriptions + assert all("description" not in config.resource.raw_config for config in typed_configs) assert tuple(config.resource.schema for config in typed_configs) == test_case.expected_schemas assert tuple(config.destination for config in typed_configs) == ( test_case.expected_destination_config, diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index 6702a216b9..104d32d16c 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -15,6 +15,7 @@ from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.errors.setting_help.main.setting_note import setting_note +from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from tests.unit.src.sqlbuild.errors.setting_help.main._test_types import ( SettingCatalogueTestCase, SettingHelpTestCase, @@ -78,6 +79,17 @@ "to skip it, add this to the MODEL header:\n MODEL (\n" " sql_analysis false,\n ...\n );", ), + SettingHelpTestCase( + "a snippet help shows every line to write", + snippet_help( + purpose="to describe it", + target="add this to seeds/lookups.yml", + lines=("- name: product_types", " description: Product type catalogue"), + ), + "to describe it, add this to seeds/lookups.yml:\n" + " - name: product_types\n" + " description: Product type catalogue", + ), SettingHelpTestCase( "several helps keep one label each", join_helps("first", "second"), diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py index eff258753e..0c5be7341e 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py @@ -36,9 +36,24 @@ expected_codes=(), ), LintNativeTestCase( - description="scenario without description does not fault", + description="scenario without description faults", contents='SCENARIO (tags ["x"]);\nSELECT 1\n', - expected_codes=(), + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="function without description faults", + contents="FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\namount > 1\n", + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="named hook without description faults", + contents="HOOK ();\nSELECT 1\n", + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="header-only check flags a model_schema model for project resolution", + contents="MODEL (model_schema order_columns);\nSELECT 1 AS order_id\n", + expected_codes=("description-present",), ), LintNativeTestCase( description="long single-line scenario description faults by formatted length", diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index ebd928eea2..bfc918855c 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -37,6 +37,16 @@ class FormatProjectTestCase: expected_formatted_count: int = 0 +@dataclass(frozen=True) +class FormatDescriptionResolutionTestCase: + """Header description faults agree with the descriptions compile resolves.""" + + description: str + project_toml: str + files: dict[str, str] + expected_description_faults: tuple[tuple[str, str], ...] + + @dataclass(frozen=True) class FormatNewlineTestCase: """Test case for preserving newline conventions while formatting.""" diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index b58d65fe4e..cf24ff4d79 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -12,6 +12,7 @@ from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from tests.unit.src.sqlbuild.lint._test_types import ( + FormatDescriptionResolutionTestCase, FormatNewlineTestCase, FormatProjectTestCase, LintBehaviorTestCase, @@ -90,10 +91,10 @@ def test_given_synthetic_project_when_linting_then_results_match_expected( expected_formatted_count=1, ), FormatProjectTestCase( - description="missing description remains a non-failing format warning", + description="missing description is a fault and the file is still formatted", files={"models/no_description.sql": NO_DESCRIPTION_MODEL}, expected_written_fragments={}, - expected_fault_codes=(), + expected_fault_codes=("description-present",), expected_formatted_count=1, ), FormatProjectTestCase( @@ -149,6 +150,64 @@ def test_given_synthetic_project_when_formatting_then_results_match_expected( assert fragment in written +@pytest.mark.parametrize( + "test_case", + [ + FormatDescriptionResolutionTestCase( + description="path default describes only the models under its path", + project_toml=PROJECT_TOML + '\n[path_defaults.marts]\ndescription = "Mart model"\n', + files={ + "models/marts/orders.sql": "MODEL ();\nSELECT 1 AS x\n", + "models/staging/stg_orders.sql": "MODEL ();\nSELECT 1 AS x\n", + }, + expected_description_faults=(("stg_orders.sql", "description-present"),), + ), + FormatDescriptionResolutionTestCase( + description="description resolution never imports project Python", + project_toml=PROJECT_TOML + '\n[path_defaults.marts]\ndescription = "Mart model"\n', + files={ + "providers/orders_api.py": 'raise RuntimeError("project Python was imported")\n', + "python/tasks/refresh.py": 'raise RuntimeError("project Python was imported")\n', + "models/marts/orders.sql": "MODEL ();\nSELECT 1 AS x\n", + "models/staging/stg_orders.sql": "MODEL ();\nSELECT 1 AS x\n", + }, + expected_description_faults=(("stg_orders.sql", "description-present"),), + ), + FormatDescriptionResolutionTestCase( + description="model schema description satisfies the header only when present", + project_toml=PROJECT_TOML, + files={ + "schemas/orders.sql": ( + 'SCHEMA (name order_shape, description "One row per order", ' + "columns (id (type INTEGER)));\n" + "SCHEMA (name bare_shape, columns (id (type INTEGER)));\n" + ), + "models/orders.sql": "MODEL (model_schema order_shape);\nSELECT 1 AS id\n", + "models/returns.sql": "MODEL (model_schema bare_shape);\nSELECT 1 AS id\n", + }, + expected_description_faults=(("returns.sql", "description-present"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_inherited_descriptions_when_formatting_then_faults_match_compile_resolution( + test_case: FormatDescriptionResolutionTestCase, tmp_path: Path +) -> None: + _ = (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + relative_path: str + contents: str + for relative_path, contents in test_case.files.items(): + target: Path = tmp_path / relative_path + _ = target.parent.mkdir(parents=True, exist_ok=True) + _ = target.write_text(contents, encoding="utf-8") + + result: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig(), write=False) + + assert tuple(sorted((fault.file_path.name, fault.code) for fault in result.faults)) == ( + test_case.expected_description_faults + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 7f6519a7dc..c31b0956bb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -60,7 +60,6 @@ "SQBRTEST102", "SQBRTEST103", "SQBRTEST104", - "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", "SQBRTEST203", @@ -79,7 +78,6 @@ "SQBRTEST102", "SQBRTEST103", "SQBRTEST104", - "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", "SQBRTEST203", diff --git a/website/examples/hero-shop/models/finance/daily_revenue.sql b/website/examples/hero-shop/models/finance/daily_revenue.sql index 21cbbb30b2..7270016cb3 100644 --- a/website/examples/hero-shop/models/finance/daily_revenue.sql +++ b/website/examples/hero-shop/models/finance/daily_revenue.sql @@ -1,4 +1,5 @@ MODEL ( + description "Daily revenue from successful payments", materialized incremental, incremental_strategy delete_insert, cursor revenue_date, diff --git a/website/examples/tidy-shop/models/daily_revenue.sql b/website/examples/tidy-shop/models/daily_revenue.sql index 6440f96b40..9ce8506dab 100644 --- a/website/examples/tidy-shop/models/daily_revenue.sql +++ b/website/examples/tidy-shop/models/daily_revenue.sql @@ -1,4 +1,5 @@ MODEL ( + description "Daily revenue from successful payments", materialized table, ); diff --git a/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql b/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql index 7f1bc18a47..a2633f291e 100644 --- a/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql +++ b/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Orders placed by one customer with their line totals", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql b/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql index 98ef857e40..a722c20280 100644 --- a/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql +++ b/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/website/examples/waffle-shop/sources/raw.yml b/website/examples/waffle-shop/sources/raw.yml index 5bdbae9c55..69654f35a3 100644 --- a/website/examples/waffle-shop/sources/raw.yml +++ b/website/examples/waffle-shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw__customers + description: Raw customer records loaded by the Python loader managed: true write_strategy: table columns: @@ -15,6 +16,7 @@ sources: type: TIMESTAMP - name: raw__orders + description: Raw order records loaded by the Python loader managed: true write_strategy: delete_insert cursor_column: ordered_at @@ -33,6 +35,7 @@ sources: type: VARCHAR - name: raw__payments + description: Payment records for each order, defined inline expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status From 3aa41186a423057ab51a19e73ba491cf69ea2a4d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:49:11 +0100 Subject: [PATCH 2/4] test: describe every fixture resource for required descriptions Add neutral generated descriptions (for example "Test model orders.") to the inline and on-disk test, script and performance-project fixtures so they compile under the P010 description requirement: MODEL, SCENARIO, FUNCTION, HOOK and SCHEMA headers, seed and source YAML entries (including dlt resources), and docstrings on Python tasks, assets, checks, loaders, hooks and UDFs. Generated benchmark projects now emit descriptions too, so their fingerprints change for this reason only. Diagnostic line expectations that moved with inserted lines are updated. --- .../_helpers/dense_project.py | 6 +- .../_helpers/diagnostic_project.py | 24 +- .../sqlbuild_project/models/local_only.sql | 1 + .../models/marts/deprecated_orders.sql | 2 +- .../models/marts/downstream_orders.sql | 1 + .../models/marts/event_time_orders.sql | 1 + .../models/marts/mart_orders.sql | 2 +- .../python/loaders/strategy_sources.py | 5 + .../source_loader_strategies/sources/raw.yml | 5 + .../functions/python/is_completed_order_py.py | 1 + .../functions/sql/customer_orders.sql | 1 + .../functions/sql/is_completed_order.sql | 1 + .../python/loaders/waffle_sources.py | 2 + .../e2e/fixtures/waffle_shop/sources/raw.yml | 3 + .../cli/commands/main/audit/helpers.py | 4 +- .../cli/commands/main/bigquery/helpers.py | 2 +- .../commands/main/bigquery/test_bigquery.py | 52 +-- .../main/bigquery/test_dlt_bigquery.py | 3 +- .../cli/commands/main/build/_test_types.py | 5 +- .../cli/commands/main/build/helpers.py | 114 +++-- .../test_build_no_tests_no_audits.py | 179 ++++---- .../main/build/own_change_replay/helpers.py | 37 +- .../main/build/test_append_cursor_build.py | 6 +- .../main/build/test_audit_factory_failures.py | 2 +- .../main/build/test_audit_failures.py | 14 +- .../cli/commands/main/build/test_build.py | 184 ++++---- .../build/test_capped_microbatch_build.py | 14 +- .../main/build/test_compile_json_behavior.py | 13 +- .../build/test_concurrent_microbatch_build.py | 22 +- .../build/test_cursor_runtime_failures.py | 8 +- .../build/test_dynamic_column_contract.py | 4 +- .../commands/main/build/test_enum_contract.py | 2 +- .../main/build/test_execution_limits.py | 8 +- .../main/build/test_explicit_references.py | 38 +- .../main/build/test_expression_sources.py | 3 +- .../main/build/test_loader_watermark_build.py | 6 +- .../build/test_manifest_artifact_gating.py | 2 +- ...test_microbatch_aware_cursor_date_input.py | 1 + .../build/test_microbatch_direct_lifecycle.py | 2 +- .../test_mixed_timestamp_grain_replay.py | 11 +- .../build/test_model_backed_cursor_build.py | 11 +- .../commands/main/build/test_python_hooks.py | 48 ++- .../main/build/test_python_node_identity.py | 5 + .../main/build/test_python_node_selection.py | 2 +- .../main/build/test_python_source_deferral.py | 14 +- .../main/build/test_query_change_tracking.py | 5 +- .../build/test_query_fingerprint_changes.py | 8 +- .../main/build/test_query_propagation.py | 23 +- .../build/test_replay_on_change_full_build.py | 13 +- .../main/build/test_reusable_model_schemas.py | 5 +- .../main/build/test_scoped_declarations.py | 10 +- .../main/build/test_seed_watermark_build.py | 4 +- .../main/build/test_snapshot_build.py | 167 ++++--- .../main/build/test_sql_hook_identity.py | 2 +- .../main/build/test_sql_hook_reads.py | 16 +- .../build/test_table_function_dependency.py | 4 +- .../main/build/test_template_expressions.py | 3 +- .../main/build/test_validation_failures.py | 12 +- .../cli/commands/main/check/helpers.py | 13 + .../cli/commands/main/clone/test_clone.py | 31 +- .../compile/fixtures/semantic/corpus.json | 408 +++++++++--------- .../fixtures/semantic/typed_orders.sql | 1 + .../cli/commands/main/compile/helpers.py | 67 +-- .../test_compile_cache_invalidation.py | 2 +- .../compile/test_compile_complexity_guard.py | 2 +- .../main/compile/test_compile_selection.py | 6 +- .../compile/test_dense_compile_performance.py | 16 +- ...fresh_process_compile_cache_performance.py | 24 +- .../test_fresh_process_compile_performance.py | 6 +- .../main/compile/test_function_header_keys.py | 4 +- .../compile/test_function_name_validation.py | 4 +- .../main/compile/test_namespace_validation.py | 52 +-- .../main/compile/test_path_defaults.py | 4 +- .../compile/test_python_project_layout.py | 22 +- .../compile/test_source_semantic_binding.py | 6 +- .../main/compile/test_unknown_config_keys.py | 18 +- .../test_varied_compile_cache_performance.py | 8 +- .../cli/commands/main/databricks/helpers.py | 3 +- .../main/databricks/test_databricks.py | 36 +- .../main/databricks/test_dlt_databricks.py | 3 +- .../sqlbuild/cli/commands/main/dbt/helpers.py | 9 +- .../cli/commands/main/dbt/test_init.py | 4 +- .../cli/commands/main/dbt/test_plan_safety.py | 3 +- .../cli/commands/main/debug/test_debug.py | 2 + .../cli/commands/main/diff/helpers.py | 11 +- .../cli/commands/main/format/helpers.py | 3 +- .../commands/main/format/test_rule_fixes.py | 14 +- .../cli/commands/main/freshness/helpers.py | 16 +- .../cli/commands/main/init/test_init.py | 6 +- .../cli/commands/main/janitor/helpers.py | 8 +- .../cli/commands/main/janitor/test_janitor.py | 5 +- .../main/janitor/test_janitor_targets.py | 3 +- .../cli/commands/main/lineage/helpers.py | 17 +- .../cli/commands/main/lineage/test_lineage.py | 4 +- .../cli/commands/main/load/helpers.py | 27 +- .../cli/commands/main/load/test_dlt.py | 9 +- .../cli/commands/main/load/test_load.py | 156 +++---- .../main/load/test_load_relation_guard.py | 8 +- .../commands/main/model_migrations/helpers.py | 11 +- .../model_migrations/test_old_name_views.py | 2 +- .../cli/commands/main/motherduck/helpers.py | 2 +- .../cli/commands/main/plan/helpers.py | 17 +- .../main/plan/test_missing_source_tables.py | 2 +- .../main/plan/test_physical_binding.py | 6 +- .../main/plan/test_removed_interfaces.py | 6 +- .../cli/commands/main/postgres/helpers.py | 14 +- .../main/postgres/test_dlt_postgres.py | 3 +- .../commands/main/postgres/test_postgres.py | 42 +- .../main/providers/test_event_exporters.py | 11 +- .../commands/main/providers/test_providers.py | 75 +++- .../cli/commands/main/refactor/helpers.py | 37 +- .../cli/commands/main/rules/helpers.py | 13 +- .../rules/test_rules_cache_invalidation.py | 2 +- .../main/rules/test_sql_quality_rules.py | 2 +- .../commands/main/rules/test_unevaluated.py | 14 +- .../main/rules/test_unused_cte_contexts.py | 14 +- .../cli/commands/main/scenario/helpers.py | 37 +- .../main/scenario/test_authored_check_sql.py | 10 +- .../commands/main/scenario/test_scenario.py | 96 +++-- .../cli/commands/main/scope/helpers.py | 2 +- .../cli/commands/main/scope/test_scope.py | 2 +- .../cli/commands/main/seed/test_seed.py | 8 +- .../cli/commands/main/snowflake/helpers.py | 3 +- .../main/snowflake/test_dlt_snowflake.py | 3 +- .../commands/main/snowflake/test_snowflake.py | 50 +-- .../main/sqlserver/test_dlt_sqlserver.py | 3 +- .../commands/main/sqlserver/test_sqlserver.py | 52 +-- .../cli/commands/main/test/helpers.py | 215 +++++---- .../main/test/test_dialect_string_escapes.py | 4 +- .../main/test/test_set_operation_expected.py | 4 +- .../commands/main/test/test_statement_size.py | 4 +- .../test/test_upstream_function_fallback.py | 8 +- .../sqlbuild/cli/commands/shared/helpers.py | 35 +- .../sqlbuild/integrations/dagster/helpers.py | 6 +- .../dagster/test_sqlbuild_assets.py | 14 +- .../integrations/ingestr/test_ingestr.py | 14 +- .../main/column_migrations/helpers.py | 4 +- .../test_column_rename_safety.py | 2 +- .../src/sqlbuild/cli/commands/main/helpers.py | 26 +- .../commands/main/model_migrations/helpers.py | 9 +- .../test_manual_migrations.py | 6 +- .../commands/main/old_name_views/helpers.py | 11 +- .../commands/main/test_build_test_planning.py | 4 +- .../cli/commands/main/test_compile.py | 24 +- .../main/test_compile_native_facts.py | 41 +- .../main/test_compile_prepared_artifacts.py | 4 +- .../commands/main/test_contract_command.py | 10 +- .../cli/commands/main/test_discovery_work.py | 2 +- .../main/test_dropped_relation_first_run.py | 8 +- .../sqlbuild/cli/commands/main/test_format.py | 12 +- .../cli/commands/main/test_load_command.py | 152 ++++++- .../sqlbuild/cli/commands/main/test_rules.py | 65 ++- .../commands/main/test_rules_concurrent.py | 4 +- .../commands/main/test_source_rebinding.py | 4 +- .../main/test_target_retention_views.py | 7 +- .../sqlbuild/compiler/pipeline/_test_types.py | 2 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 11 +- .../pipeline/test_attached_audit_gates.py | 6 +- .../compiler/pipeline/test_audit_factories.py | 4 +- .../compiler/pipeline/test_binding_waves.py | 10 +- .../pipeline/test_completed_shape_cache.py | 10 +- .../compiler/pipeline/test_diff_selectors.py | 10 +- .../pipeline/test_expression_catalog_cache.py | 10 +- .../test_macro_declaration_context.py | 10 +- .../pipeline/test_macro_load_count.py | 4 +- .../sqlbuild/compiler/pipeline/test_main.py | 54 ++- .../pipeline/test_measurement_audits.py | 12 +- .../pipeline/test_native_binding_catalog.py | 24 +- .../pipeline/test_python_source_reads.py | 8 +- .../compiler/pipeline/test_review_bindings.py | 26 +- .../pipeline/test_selection_lineage.py | 6 +- .../pipeline/test_semantic_binding.py | 40 +- .../pipeline/test_semantic_diagnostic_ux.py | 19 +- .../compiler/pipeline/test_semantic_scopes.py | 10 +- .../test_semantic_snowflake_release.py | 8 +- .../compiler/pipeline/test_semantic_triage.py | 40 +- .../pipeline/test_set_operation_types.py | 9 +- .../pipeline/test_shared_binding_queries.py | 43 +- .../test_sql_test_input_extraction.py | 4 +- .../sql_test_assembly/test_execute_chain.py | 6 +- .../planner/_helpers/test_fixture_planning.py | 8 +- .../sqlbuild/compiler/planner/main/helpers.py | 6 +- .../build/concurrent/test_concurrent_build.py | 49 ++- .../concurrent/test_microbatch_concurrency.py | 5 +- .../concurrent/test_ordering_invariant.py | 14 +- .../test_stateless_sequential_microbatch.py | 3 +- .../test_advanced_custom_materialization.py | 8 +- .../src/sqlbuild/executor/build/helpers.py | 15 +- .../build/snowflake_metadata/helpers.py | 11 +- .../executor/build/test_build_execution.py | 132 +++--- .../build/test_measurement_audit_execution.py | 2 +- .../executor/build/test_snapshot_execution.py | 220 +++++----- .../executor/build/test_table_type_routing.py | 4 +- .../src/sqlbuild/executor/testing/helpers.py | 12 +- .../src/sqlbuild/integrations/dbt/helpers.py | 2 +- .../integrations/dbt/test_manifest.py | 2 +- .../sqlbuild/integrations/dbt/test_plan.py | 34 +- .../adapters/snowflake/inspection/helpers.py | 5 +- .../commands/_helpers/lineage/test_cache.py | 2 +- .../_helpers/scenario_execution/helpers.py | 2 +- .../cli/commands/main/compile/helpers.py | 13 +- .../main/compile/test_compile_command.py | 10 +- .../sqlbuild/cli/commands/main/dag/helpers.py | 8 +- .../cli/commands/main/dag/test_dag_command.py | 1 + .../compiler/compile/_helpers/helpers.py | 8 +- .../compile/_helpers/test_analysis_cache.py | 51 ++- .../compile/_helpers/test_assembly.py | 37 +- .../_helpers/test_attached_audit_gates.py | 14 +- .../compile/_helpers/test_audit_factories.py | 4 +- .../_helpers/test_declaration_scopes.py | 104 ++--- .../compile/_helpers/test_declarations.py | 58 +-- .../compile/_helpers/test_hook_reads.py | 20 +- .../compile/_helpers/test_model_schemas.py | 24 +- .../_helpers/test_parameterized_sql_tests.py | 12 +- .../compile/_helpers/test_path_defaults.py | 4 +- .../_helpers/test_python_sql_references.py | 10 +- .../test_scoped_named_declarations.py | 20 +- .../_helpers/test_typed_macro_references.py | 6 +- .../compiler/compile/test_cursor_start.py | 8 +- .../sqlbuild/compiler/compile/test_main.py | 339 ++++++++------- .../compile/test_resolved_connection.py | 4 +- .../_helpers/test_repository_edits.py | 2 +- .../compiler/contracts/test_validate.py | 2 +- .../sqlbuild/compiler/manifest/test_main.py | 2 +- .../warehouse/test_semantic_sources.py | 8 +- .../python_nodes/_helpers/test_identity.py | 1 + .../src/sqlbuild/compiler/scopes/helpers.py | 4 +- .../compiler/scopes/test_offline_cache.py | 18 +- .../executor/clone/_helpers/helpers.py | 2 +- .../executor/pipeline/_helpers/helpers.py | 2 +- .../src/sqlbuild/integrations/dbt/helpers.py | 2 +- .../lint/_helpers/test_header_wrapping.py | 16 +- .../sqlbuild/lint/_helpers/test_headers.py | 20 +- .../unit/src/sqlbuild/lint/test_cli_format.py | 4 +- .../provider/_helpers/test_injection.py | 2 +- .../main/evaluate_rule/test_evaluate_rule.py | 6 +- .../main/run_rules/test_run_rules.py | 2 +- 237 files changed, 2973 insertions(+), 2219 deletions(-) diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index a0ca920076..0d7ef833c5 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -156,6 +156,7 @@ def _write_sources(*, project_dir: Path, source_count: int) -> None: entries.extend( ( f" - name: orders_{index:05d}", + f" description: Generated source orders_{index:05d}.", " contract: enforced", f' expression: "(SELECT 1 AS id, CAST(1 AS DOUBLE) AS amount{extra})"', " columns:", @@ -248,9 +249,8 @@ def _model_sql(*, index: int, source_count: int, function_count: int, macro_coun f'description "Order quantity measure"{audit})' ) header: str = ( - "MODEL (materialized table, contract enforced, columns (\n" - + ",\n".join(declarations) - + "\n));\n" + 'MODEL (description "Generated dense model.", materialized table, contract enforced, ' + "columns (\n" + ",\n".join(declarations) + "\n));\n" ) source_index: int = _source_index(index=index, source_count=source_count) relation: str = ( diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_project.py b/scripts/cold_compile_performance/_helpers/diagnostic_project.py index 93d4ffbe2b..71a8d54044 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_project.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_project.py @@ -16,7 +16,8 @@ def write_diagnostic_project( columns: list[str] = [f"amount_{index:03}" for index in range(width)] declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) (models / "orders.sql").write_text( - f"MODEL (materialized view, contract enforced, columns ({declarations}));\nSELECT " + "MODEL (description 'Test model orders.', " + f"materialized view, contract enforced, columns ({declarations}));\nSELECT " + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) ) for model_index in range(model_count): @@ -31,12 +32,14 @@ def write_diagnostic_project( conversions: list[str] = [ f"{columns[index % width]} = 'shipped' AS risky_{index:03}" for index in range(50) ] - sql: str = "MODEL (materialized view);\nWITH\n" + ",\n".join(ctes) + sql: str = "MODEL (description 'Test model.', materialized view);\nWITH\n" + ",\n".join( + ctes + ) sql += "\nSELECT\n" + ",\n".join((*errors, *conversions)) + f"\nFROM {previous}\n" name = f"large_orders_{model_index}" (models / f"{name}.sql").write_text(sql) (models / f"downstream_orders_{model_index}.sql").write_text( - "MODEL (materialized view);\nSELECT " + "MODEL (description 'Test model.', materialized view);\nSELECT " + ", ".join(f"broken_{index:03} + 1 AS result_{index:03}" for index in range(50)) + f' FROM __ref("{name}")' ) @@ -52,9 +55,13 @@ def write_cascade_project(*, project_dir: Path, depth: int, width: int = 50) -> ) columns: list[str] = [f"amount_{index:03}" for index in range(width)] declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) - header: str = f"MODEL (materialized view, contract enforced, columns ({declarations}));\n" + header: str = ( + "MODEL (description 'Test model.', " + f"materialized view, contract enforced, columns ({declarations}));\n" + ) (models / "input_orders.sql").write_text( - f"MODEL (materialized view, contract enforced, columns ({declarations}, " + "MODEL (description 'Test model input_orders.', " + f"materialized view, contract enforced, columns ({declarations}, " "ordered_at (type TIMESTAMP)));\nSELECT " + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) + ", CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" @@ -94,7 +101,7 @@ def write_function_name_project(*, project_dir: Path, model_count: int, calls: i ) ) (models / f"wide_products_{model_index}.sql").write_text( - "MODEL (database warehouse, schema analytics);\nSELECT\n" + "MODEL (description 'Test model.', database warehouse, schema analytics);\nSELECT\n" + ",\n".join(projections) + "\nFROM (SELECT 'anvil' AS sku) AS products\nWHERE NOT (sku IN ('a', 'b'))\n" ) @@ -109,7 +116,8 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int 'name = "dense_diagnostic_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' ) (models / "orders.sql").write_text( - "MODEL (materialized view, contract enforced, columns (order_id (type INTEGER), " + "MODEL (description 'Test model orders.', " + "materialized view, contract enforced, columns (order_id (type INTEGER), " "status (type VARCHAR)));\nSELECT CAST(1 AS INTEGER) AS order_id, " "CAST('open' AS VARCHAR) AS status" ) @@ -120,7 +128,7 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int for index in range(diagnostics) ] (models / "order_diagnostics.sql").write_text( - "MODEL (materialized view);\nSELECT\n" + "MODEL (description 'Test model order_diagnostics.', materialized view);\nSELECT\n" + ",\n".join(projections) + '\nFROM __ref("orders") AS o\n' ) diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql index 574a992da8..89f0316c76 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model local_only.", tags [sqb_only], columns (order_id (audits [not_null])), ); diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql index 54556ccfa2..866ad7fbeb 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql @@ -1,3 +1,3 @@ -MODEL (tags [sqb_only, deprecated]); +MODEL (description "Test model deprecated_orders.", tags [sqb_only, deprecated]); select order_id from __ref("local_only") diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql index afdb4c8278..ce18b32233 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model downstream_orders.", tags [nightly, finance], columns (order_id (audits [not_null])), ); diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql index bb4ca7aa11..5940f2ab52 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model event_time_orders.", materialized incremental, incremental_strategy delete_insert, cursor ordered_at, diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql index dde653a700..eb082857ee 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql @@ -1,3 +1,3 @@ -MODEL (tags [marts]); +MODEL (description "Test model mart_orders.", tags [marts]); select order_id from __ref("downstream_orders") diff --git a/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py b/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py index b666d77552..00846041ce 100644 --- a/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py +++ b/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py @@ -6,6 +6,7 @@ @loader def raw_countries(ctx: object) -> list[dict[str, object]]: + """Test loader raw_countries.""" _ = ctx return [ {"country_id": 1, "country_code": "US", "country_name": "United States"}, @@ -15,6 +16,7 @@ def raw_countries(ctx: object) -> list[dict[str, object]]: @loader def raw_webhook_events(ctx: object) -> list[dict[str, object]]: + """Test loader raw_webhook_events.""" _ = ctx return [ {"event_id": 101, "event_name": "signup"}, @@ -24,6 +26,7 @@ def raw_webhook_events(ctx: object) -> list[dict[str, object]]: @loader def raw_order_events(ctx: Any) -> list[dict[str, object]]: + """Test loader raw_order_events.""" if ctx.current_cursor_value is None: return [ { @@ -53,6 +56,7 @@ def raw_order_events(ctx: Any) -> list[dict[str, object]]: @loader def raw_customers(ctx: Any) -> list[dict[str, object]]: + """Test loader raw_customers.""" if ctx.current_cursor_value is None: return [ { @@ -82,6 +86,7 @@ def raw_customers(ctx: Any) -> list[dict[str, object]]: @loader def raw_loader_status(ctx: Any) -> None: + """Test loader raw_loader_status.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " diff --git a/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml b/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml index 95489d83b3..e8936f12cf 100644 --- a/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml +++ b/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_countries + description: Test source raw_countries. managed: true write_strategy: table columns: @@ -11,6 +12,7 @@ sources: type: VARCHAR - name: raw_webhook_events + description: Test source raw_webhook_events. managed: true write_strategy: append columns: @@ -20,6 +22,7 @@ sources: type: VARCHAR - name: raw_order_events + description: Test source raw_order_events. managed: true write_strategy: delete_insert cursor_column: event_at @@ -32,6 +35,7 @@ sources: type: INTEGER - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: merge unique_key: customer_id @@ -45,6 +49,7 @@ sources: type: TIMESTAMP - name: raw_loader_status + description: Test source raw_loader_status. managed: true columns: - name: status_id diff --git a/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py b/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py index 219b6f23f1..c8e9572932 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py +++ b/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py @@ -7,4 +7,5 @@ runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Test function main.""" return order_status == "completed" diff --git a/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql b/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql index 7f1bc18a47..c6c3d4e447 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql +++ b/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql b/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql index 98ef857e40..c9af25b3bf 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql +++ b/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Test function is_completed_order.", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py b/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py index a3c115f2ad..5c240440ca 100644 --- a/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py +++ b/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py @@ -57,12 +57,14 @@ @loader def raw_customers(ctx: object) -> list[dict[str, object]]: + """Test loader raw_customers.""" _ = ctx return CUSTOMERS @loader def raw_orders(ctx: object) -> Iterator[dict[str, object]]: + """Test loader raw_orders.""" _ = ctx for order_id, customer_id, waffle_type_id, quantity, ordered_at, status in ORDERS: yield { diff --git a/tests/e2e/fixtures/waffle_shop/sources/raw.yml b/tests/e2e/fixtures/waffle_shop/sources/raw.yml index 15d8b68fbe..702c0ee449 100644 --- a/tests/e2e/fixtures/waffle_shop/sources/raw.yml +++ b/tests/e2e/fixtures/waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -15,6 +16,7 @@ sources: type: TIMESTAMP - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: delete_insert cursor_column: ordered_at @@ -33,6 +35,7 @@ sources: type: VARCHAR - name: raw_payments + description: Test source raw_payments. expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py index 9741150667..d44567130e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py @@ -34,7 +34,7 @@ def delivery_model_files() -> dict[str, str]: return { "models/customers.sql": dedent( """ - MODEL ( + MODEL (description "Test model customers.", materialized table, columns ( customer_id (audits [not_null]), @@ -45,7 +45,7 @@ def delivery_model_files() -> dict[str, str]: """ ).lstrip(), "models/orders.sql": ( - "MODEL (\n materialized table,\n columns (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n columns (\n" f"{columns}\n" " ),\n);\n\n" f"SELECT\n customer_id,\n {selected}\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py index 2619ba35ad..dd37a3407e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py @@ -258,7 +258,7 @@ def prepare_bigquery_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: models_dir: Path = project_dir / "models" / "staging" models_dir.mkdir(parents=True, exist_ok=True) (models_dir / "stg_orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model stg_orders.',\n" " materialized table,\n" " unique_key [order_id],\n" " cursor order_id,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index 35065422c2..ede85b5f93 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -172,14 +172,14 @@ def test_given_python_result_when_running_check_on_bigquery_then_persists_node_r "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -253,12 +253,12 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {dataset_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -350,7 +350,7 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro BigQueryScenarioLocalReplayE2ETestCase( description="captures bigquery fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " TIMESTAMP_TRUNC(event_ts, DAY) AS event_day,\n" @@ -360,7 +360,7 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro "GROUP BY customer_id, TIMESTAMP_TRUNC(event_ts, DAY)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TIMESTAMP '2026-01-01 08:15:00 UTC' " @@ -391,12 +391,12 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro description="reports bigquery local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, SAFE_CAST(amount_text AS INT64) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, '1500' AS amount_text\n" @@ -419,14 +419,14 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro description="reports bigquery local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(SAFE_CAST(amount_text AS INT64)) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, '1500' AS amount_text\n" @@ -535,7 +535,7 @@ def test_given_source_deferral_env_when_building_on_bigquery_then_reads_prod_and "sqlbuild_local.toml": build_bigquery_local_config(location=location), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -548,10 +548,10 @@ def test_given_source_deferral_env_when_building_on_bigquery_then_reads_prod_and "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -1111,7 +1111,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_bigquery_th repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1125,7 +1125,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_bigquery_th "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -1191,7 +1191,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1206,10 +1206,10 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1266,7 +1266,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1275,7 +1275,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1301,7 +1301,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1312,7 +1312,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1333,7 +1333,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1344,7 +1344,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1369,7 +1369,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_bigquery_then repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1591,7 +1591,7 @@ def test_given_bigquery_invalid_model_when_building_then_underlying_error_is_pre project_dir, dataset_name = prepare_bigquery_waffle_shop(tmp_path=tmp_path) broken_model: Path = project_dir / "models" / "marts" / "bigquery_broken_model.sql" broken_model.write_text( - "MODEL (materialized table);\n\nSELECT missing_column FROM UNNEST([STRUCT(1 AS id)])", + "MODEL (description 'Test model bigquery_broken_model.', materialized table);\n\nSELECT missing_column FROM UNNEST([STRUCT(1 AS id)])", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py index e141f10e4c..e900cbb35c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py @@ -66,11 +66,12 @@ def test_given_dlt_source_when_loading_to_bigquery_then_table_is_materialized_an f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 095d87af2d..f47dc5c487 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1190,7 +1190,10 @@ class PythonNodeSelectionE2ETestCase: expected_row_counts: dict[str, int] expected_node_counts: dict[str, int] expected_missing_tables: tuple[str, ...] = () - orders_sql: str = 'MODEL (materialized table);\nSELECT order_id FROM __source("raw_orders")\n' + orders_sql: str = ( + 'MODEL (description "Test model.", ' + 'materialized table);\nSELECT order_id FROM __source("raw_orders")\n' + ) existing_sql: str = "" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 60c5479ca0..5347d40f26 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -55,6 +55,7 @@ def capped_microbatch_project_files( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -62,7 +63,7 @@ def capped_microbatch_project_files( + "\n", "models/capped_events.sql": dedent( f""" - MODEL ( + MODEL (description "Test model capped_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -90,7 +91,7 @@ def capped_microbatch_project_files( + "\n", "models/downstream_events.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model downstream_events.", materialized view); SELECT id, event_time FROM __ref("capped_events") """ @@ -106,7 +107,7 @@ def capped_watermark_consumer_project_files(*, limit_action: str) -> dict[str, s repo_files["models/downstream_events.sql"] = ( dedent( """ - MODEL ( + MODEL (description "Test model downstream_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -138,7 +139,7 @@ def capped_filter_join_consumer_project_files(*, limit_action: str) -> dict[str, repo_files["models/downstream_events.sql"] = ( dedent( """ - MODEL ( + MODEL (description "Test model downstream_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -285,7 +286,7 @@ def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, columns (order_id (audits [not_null])), ); @@ -321,6 +322,7 @@ def build_freshness_error_branch_source_yml( f""" sources: - name: raw_orders + description: Test source raw_orders. expression: SELECT {order_id} AS order_id freshness: strategy: sql @@ -329,6 +331,7 @@ def build_freshness_error_branch_source_yml( age_policy: error_after: 1h - name: raw_customers + description: Test source raw_customers. expression: SELECT {customer_id} AS customer_id freshness: strategy: sql @@ -348,7 +351,7 @@ def replay_microbatch_model_sql(*, value_expression: str, replay_policy: str = " return ( dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -392,7 +395,10 @@ def direct_microbatch_project_toml( def raw_events_source_yml() -> str: """Return the canonical raw-events source declaration.""" - return "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + return ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" + ) def timestamp_microbatch_model_sql( @@ -407,7 +413,7 @@ def timestamp_microbatch_model_sql( return ( dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -469,10 +475,11 @@ def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, '[connection]\ndatabase = "dropped_orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" f" incremental_strategy {incremental_strategy},\n" " unique_key id,\n" @@ -496,30 +503,36 @@ def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, _GATE_TARGET_FILES: dict[str, dict[str, str]] = { "model": { "models/orders.sql": ( - "MODEL (materialized table, audits [code_check (severity error)]);\n\n" + "MODEL (description 'Test model orders.', " + "materialized table, audits [code_check (severity error)]);\n\n" "SELECT '{code}' AS code\n" ), "models/order_summary.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __ref("orders")\n' + 'MODEL (description "Test model order_summary.", ' + 'materialized table);\n\nSELECT code FROM __ref("orders")\n' ), }, "source": { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" " audits:\n - code_check:\n severity: error\n" ), "models/staged_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __source("raw_orders")\n' + 'MODEL (description "Test model staged_orders.", ' + 'materialized table);\n\nSELECT code FROM __source("raw_orders")\n' ), }, "seed": { "seeds/order_codes.yml": ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n" + " columns:\n - name: code\n" " type: VARCHAR\n audits:\n - code_check:\n severity: error\n" ), "seeds/order_codes.csv": "code\n{code}\n", "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", ' + 'materialized table);\n\nSELECT code FROM __seed("order_codes")\n' ), }, } @@ -542,7 +555,8 @@ def prepare_attached_audit_gate_project( ), "audits/generic/code_check.sql": _GATE_AUDIT_TEMPLATE.format(read=read), "models/valid_codes.sql": ( - "MODEL (materialized table);\n\nSELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" + "MODEL (description 'Test model valid_codes.', " + "materialized table);\n\nSELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" ), } files.update( @@ -575,7 +589,8 @@ def prepare_nested_source_gate_project(*, tmp_path: Path, raw_code: str) -> Path f'[connection]\ndatabase = "{ATTACHED_AUDIT_GATE_DATABASE}"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_codes\n schema: main\n table: raw_codes\n" + "sources:\n - name: raw_codes\n description: Test source raw_codes.\n" + " schema: main\n table: raw_codes\n" " audits:\n - source_check:\n severity: error\n" ), "audits/generic/source_check.sql": _GATE_AUDIT_TEMPLATE.format( @@ -585,11 +600,12 @@ def prepare_nested_source_gate_project(*, tmp_path: Path, raw_code: str) -> Path read='__source("raw_codes")' ), "models/orders.sql": ( - "MODEL (materialized table, audits [order_check (severity error)]);\n\n" + "MODEL (description 'Test model orders.', " + "materialized table, audits [order_check (severity error)]);\n\n" "SELECT 'A' AS code\n" ), "models/valid_codes.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model valid_codes.', materialized table);\n\n" "SELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" ), }, @@ -623,29 +639,35 @@ def build_asset_names(stdout: str) -> tuple[str, ...]: _AUDIT_ERROR_TARGET_FILES: dict[str, dict[str, str]] = { "source": { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" " audits:\n - broken_check:\n severity: {severity}\n" ), "audits/generic/broken_check.sql": _BROKEN_AUDIT, "models/staged_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __source("raw_orders")\n' + 'MODEL (description "Test model staged_orders.", ' + 'materialized table);\n\nSELECT code FROM __source("raw_orders")\n' ), }, "seed": { "seeds/order_codes.yml": ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n" + " columns:\n - name: code\n" " type: VARCHAR\n audits:\n - broken_check:\n" " severity: {severity}\n" ), "seeds/order_codes.csv": "code\nA\n", "audits/generic/broken_check.sql": _BROKEN_AUDIT, "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", ' + 'materialized table);\n\nSELECT code FROM __seed("order_codes")\n' ), }, "end": { - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 'A' AS code\n", - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 'A' AS code\n", + "models/orders.sql": "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT 'A' AS code\n", + "models/customers.sql": "MODEL (description 'Test model customers.', " + "materialized table);\n\nSELECT 'A' AS code\n", "audits/singular/broken_check.sql": ( "AUDIT (severity {severity});\n\n" 'SELECT o.code FROM __ref("orders") o JOIN __ref("customers") c USING (code)\n' @@ -704,12 +726,14 @@ def prepare_audit_read_plan_project(*, tmp_path: Path) -> Path: '[connection]\ndatabase = "plan_shop.duckdb"\n' ), "seeds/waffle_types.yml": ( - "seeds:\n - name: waffle_types\n columns:\n" + "seeds:\n - name: waffle_types\n description: Test seed waffle_types.\n" + " columns:\n" " - name: waffle_type_id\n type: INTEGER\n" ), "seeds/waffle_types.csv": "waffle_type_id\n1\n", "models/stg_orders.sql": ( - "MODEL (\n materialized table,\n columns (\n waffle_type_id (\n" + "MODEL (description 'Test model stg_orders.'," + "\n materialized table,\n columns (\n waffle_type_id (\n" ' audits [relationships (to __seed("waffle_types"), field waffle_type_id)],\n' " ),\n ),\n);\n\nSELECT 1 AS waffle_type_id\n" ), @@ -731,7 +755,7 @@ def prepare_audit_read_plan_project(*, tmp_path: Path) -> Path: "from sqlbuild.refs import model\n\n\n" '@hook(reads=model("stg_customers"))\n' "def record_customers(ctx):\n" - ' customers = ctx.relation(model("stg_customers"))\n' + ' """Test hook record_customers."""\n customers = ctx.relation(model("stg_customers"))\n' " ctx.execute_sql(\n" ' f"CREATE OR REPLACE TABLE customer_counts AS SELECT count(*) AS n FROM {customers}"\n' " )\n" @@ -750,20 +774,24 @@ def explicit_reference_project_files( '[connection]\ndatabase = "warehouse.duckdb"\n' + references ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": ( - "MODEL (materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" + "MODEL (description 'Test model stg_orders_eu.', " + "materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": ( - "MODEL (materialized table);\nSELECT 2 AS order_id, 20 AS amount\n" + "MODEL (description 'Test model stg_orders_us.', " + "materialized table);\nSELECT 2 AS order_id, 20 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_customers.sql": ( - "MODEL (materialized table);\nSELECT 1 AS customer_id\n" + "MODEL (description 'Test model stg_customers.', " + "materialized table);\nSELECT 1 AS customer_id\n" ), f"{EXPLICIT_REFERENCE_MODELS}/all_orders.sql": ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model all_orders.', materialized table);\n" '@union_all([__ref("stg_orders_eu"), __ref("stg_orders_us")])\n' ), f"{EXPLICIT_REFERENCE_MODELS}/orders_summary.sql": ( - 'MODEL (materialized table, post_hooks [python("record_customers")]);\n' + 'MODEL (description "Test model orders_summary.", ' + 'materialized table, post_hooks [python("record_customers")]);\n' 'SELECT count(*) AS order_count FROM __ref("all_orders")\n' ), EXPLICIT_REFERENCE_MACRO_PATH: EXPLICIT_REFERENCE_MACROS, @@ -778,10 +806,10 @@ def explicit_reference_literal_loader(*, table: str) -> str: "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" "@loader\n" "def raw_customers(ctx):\n" - f' ctx.query("SELECT count(*) FROM {table}")\n' + f' """Test loader raw_customers."""\n ctx.query("SELECT count(*) FROM {table}")\n' " return [{'id': 1}]\n" ) @@ -793,6 +821,7 @@ def _counting_python_node(*, decorator: str, name: str, ref: str, returned: str) return ( f"@{decorator}(depends_on={ref})\n" f"def {name}(ctx):\n" + f' """Test {decorator} {name}."""\n' f" relation = ctx.relation({ref})\n" f' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.{name}_result AS ' f'SELECT count(*) AS n FROM {{relation}}")\n' @@ -828,13 +857,17 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_customers\n description: Test source raw_customers.\n" + " managed: true\n write_strategy: table\n" " columns:\n - name: customer_id\n type: INTEGER\n" ), "python/loaders/customers.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_customers(ctx):\n return [{'customer_id': 99}]\n" + "@loader\ndef raw_customers(ctx):\n" + " '''Test loader raw_customers.'''\n" + " return [{'customer_id': 99}]\n" ), "python/tasks/counts.py": ( "from sqlbuild.refs import model, source\nfrom sqlbuild.tasks import task\n\n\n" @@ -846,6 +879,7 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: ), "models/orders.sql": orders_sql, "models/order_summary.sql": ( - 'MODEL (materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' + 'MODEL (description "Test model order_summary.", ' + 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index d9235b499f..78727ef17e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -73,6 +73,7 @@ def test_given_project_with_python_hooks_when_running_run_then_hooks_execute( @hook def log_hook(ctx: HookContext): + '''Test hook log_hook.''' ctx.execute_sql( f"CREATE TABLE IF NOT EXISTS {ctx.destination.schema}.hook_log " "(model_name VARCHAR, phase VARCHAR)" @@ -86,7 +87,7 @@ def log_hook(ctx: HookContext): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -202,7 +203,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -210,7 +211,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/factories/generated.py": """ from pathlib import Path @@ -228,21 +229,25 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ def generated_pipeline(): @task(name="prepare_orders", tags=("factory", "runtime")) def prepare(ctx): + '''Test task prepare.''' PROJECT_DIR.joinpath("prepare_marker.txt").write_text("ran", encoding="utf-8") return ctx.result(payload={"rows": 1}) @loader(name="raw_orders", depends_on=[prepare]) def load(ctx): + '''Test loader load.''' PROJECT_DIR.joinpath("loader_marker.txt").write_text("ran", encoding="utf-8") return [{"order_id": 1}] @asset(name="orders_export", depends_on=prepare, tags=("factory", "runtime")) def export(ctx): + '''Test asset export.''' PROJECT_DIR.joinpath("asset_marker.txt").write_text("ran", encoding="utf-8") return ctx.result(materialized=True) @check(name="orders_export_check", depends_on=export, tags=("factory", "quality")) def export_check(ctx): + '''Test check export_check.''' return ctx.pass_(message="generated export exists") return [prepare, load, export, export_check] @@ -365,14 +370,14 @@ def test_given_existing_intermediate_target_when_running_source_only_then_reuses " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) setup_result: subprocess.CompletedProcess[str] = run_sqb( @@ -426,7 +431,7 @@ def test_given_task_selector_when_running_run_then_task_executes( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" + " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'})\n" ), }, @@ -481,7 +486,7 @@ def test_given_asset_selector_when_running_run_then_asset_executes( "from sqlbuild.assets import asset\n\n" "@asset\n" "def prepared_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('asset_marker.txt').write_text('ran')\n" + " '''Test asset prepared_orders.'''\n Path(__file__).parents[2].joinpath('asset_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'}, materialized=True)\n" ), }, @@ -536,7 +541,7 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result(metadata={'rows': 3})\n" + " '''Test task prepare_orders.'''\n return ctx.result(metadata={'rows': 3})\n" ), }, ) @@ -606,7 +611,7 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( "from sqlbuild.tasks import task\n\n" "@task\n" "def fail_orders(ctx):\n" - " raise RuntimeError('API unavailable')\n" + " '''Test task fail_orders.'''\n raise RuntimeError('API unavailable')\n" ), }, ) @@ -675,7 +680,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" + " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" " return ctx.result(metadata={'prepared': True})\n" ), "python/loaders/orders.py": ( @@ -684,14 +689,14 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from python.tasks.orders import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('orders_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('orders_ready.txt')\n" " if not marker.exists():\n" " raise RuntimeError('orders were not prepared')\n" " return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -701,7 +706,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ " type: INTEGER\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ), }, @@ -750,7 +755,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 100 AS amount_cents\n" " columns:\n" " - name: order_id\n" @@ -759,7 +764,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built " type: INTEGER\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ), "python/tasks/orders.py": ( @@ -768,7 +773,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def summarize_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test task summarize_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('summary.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" @@ -825,14 +830,14 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -840,7 +845,7 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads "from sqlbuild.refs import model\n\n" "@asset(depends_on=model('fact_orders'))\n" "def export_fact_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test asset export_fact_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('export.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" @@ -899,7 +904,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" - " return ctx.result(payload={'rows': 1})\n" + " '''Test task fetch_orders.'''\n return ctx.result(payload={'rows': 1})\n" ), "python/tasks/notify_orders.py": ( "from pathlib import Path\n" @@ -907,7 +912,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from sqlbuild.tasks import task\n\n" "@task(depends_on=publish_orders)\n" "def notify_orders(ctx):\n" - " metadata = ctx.result_of(node_function=publish_orders).metadata\n" + " '''Test task notify_orders.'''\n metadata = ctx.result_of(node_function=publish_orders).metadata\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(metadata['published']))\n" " return ctx.result()\n" @@ -917,7 +922,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from python.tasks.fetch_orders import fetch_orders\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" - " payload = ctx.result_of(node_function=fetch_orders).payload\n" + " '''Test asset publish_orders.'''\n payload = ctx.result_of(node_function=fetch_orders).payload\n" " return ctx.result(payload=payload, metadata={'published': True})\n" ), }, @@ -964,11 +969,11 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -982,7 +987,7 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def summarize_loaded_orders(ctx):\n" - " orders = ctx.relation(source('raw_orders'))\n" + " '''Test task summarize_loaded_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), @@ -991,7 +996,7 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from python.tasks.orders import summarize_loaded_orders\n\n" "@asset(depends_on=summarize_loaded_orders)\n" "def publish_loaded_orders(ctx):\n" - " payload = ctx.result_of(node_function=summarize_loaded_orders).payload\n" + " '''Test asset publish_loaded_orders.'''\n payload = ctx.result_of(node_function=summarize_loaded_orders).payload\n" " return ctx.result(payload=payload, materialized=False)\n" ), }, @@ -1051,13 +1056,13 @@ def test_given_skip_and_asset_selection_with_json_when_running_run_then_json_rec "from sqlbuild.tasks import task\n\n" "@task\n" "def optional_orders(ctx):\n" - " return ctx.skip(reason='no files')\n\n" + " '''Test task optional_orders.'''\n return ctx.skip(reason='no files')\n\n" ), "python/assets/orders.py": ( "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" - " return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" + " '''Test asset observed_orders.'''\n return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" ), }, ) @@ -1126,25 +1131,25 @@ def test_given_independent_python_and_sql_selectors_when_running_run_then_all_br ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __source("raw_orders")\n' ), "python/tasks/branches.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def branch_a(ctx):\n" - " Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" + " '''Test task branch_a.'''\n Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" " return ctx.result()\n\n" "@task\n" "def branch_b(ctx):\n" - " Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" + " '''Test task branch_b.'''\n Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" " return ctx.result()\n" ), }, @@ -1198,7 +1203,7 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 100 AS amount_cents\n" " columns:\n" " - name: order_id\n" @@ -1207,17 +1212,17 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def profile_stg_orders(ctx):\n" - " orders = ctx.relation(model('stg_orders'))\n" + " '''Test task profile_stg_orders.'''\n orders = ctx.relation(model('stg_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows})\n" ), @@ -1275,11 +1280,11 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1293,7 +1298,7 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" - " orders = ctx.relation(source('raw_orders'))\n" + " '''Test task profile_raw_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows})\n" ), @@ -1349,24 +1354,24 @@ def test_given_sql_ready_task_fails_when_running_run_then_footer_json_and_exit_f ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def fail_after_stg_orders(ctx):\n" - " raise RuntimeError('profile failed')\n" + " '''Test task fail_after_stg_orders.'''\n raise RuntimeError('profile failed')\n" ), }, ) @@ -1431,11 +1436,11 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " columns:\n" " - name: order_id\n" @@ -1446,7 +1451,7 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej "from sqlbuild.tasks import task\n\n" "@task(depends_on=raw_orders)\n" "def summarize_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task summarize_orders.'''\n return ctx.result()\n" ), }, ) @@ -1506,14 +1511,14 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo " ],\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "python/tasks/orders.py": ( "from python.loaders.orders import stage_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=stage_orders)\n" "def summarize_stage_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" + " '''Test task summarize_stage_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), "python/assets/orders.py": ( @@ -1521,7 +1526,7 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo "from python.tasks.orders import summarize_stage_orders\n\n" "@asset(depends_on=summarize_stage_orders)\n" "def publish_stage_orders(ctx):\n" - " payload = ctx.result_of(node_function=summarize_stage_orders).payload\n" + " '''Test asset publish_stage_orders.'''\n payload = ctx.result_of(node_function=summarize_stage_orders).payload\n" " return ctx.result(payload=payload, materialized=True)\n" ), }, @@ -1584,7 +1589,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/tasks/orders.py": ( "from pathlib import Path\n" @@ -1592,7 +1597,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_raw_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test task prepare_raw_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" ), @@ -1602,14 +1607,14 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_raw_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('window orders were not prepared')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1617,7 +1622,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1666,7 +1671,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" - " return ctx.result(payload={'order_id': 1})\n" + " '''Test task fetch_orders.'''\n return ctx.result(payload={'order_id': 1})\n" ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -1674,7 +1679,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.assets import asset\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" - " payload = ctx.result_of(node_function=fetch_orders).payload\n" + " '''Test asset publish_orders.'''\n payload = ctx.result_of(node_function=fetch_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1685,14 +1690,14 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset was not ready')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1700,7 +1705,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1753,7 +1758,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -1761,7 +1766,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "from sqlbuild.assets import asset\n\n" "@asset(depends_on=load_window_orders)\n" "def prepare_asset_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test asset prepare_asset_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('asset_ready.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" ), @@ -1771,14 +1776,14 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_asset_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset orders were not prepared')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1786,7 +1791,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1839,19 +1844,19 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/loaders/raw.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(load_window_orders,))\n" "def raw_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test loader raw_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " return [{'order_id': rows}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1859,7 +1864,7 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1912,14 +1917,14 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 7}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 7}]\n" ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" - " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" + " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), "python/assets/prepare.py": ( @@ -1928,7 +1933,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" - " payload = ctx.result_of(node_function=prepare_orders).payload\n" + " '''Test asset publish_prepared_orders.'''\n payload = ctx.result_of(node_function=prepare_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1939,12 +1944,12 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1952,14 +1957,14 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test task profile_fact_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), @@ -1968,7 +1973,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" + " '''Test asset export_fact_orders.'''\n payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), "python/tasks/notify.py": ( @@ -1977,7 +1982,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=export_fact_orders).payload\n" + " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" @@ -2050,7 +2055,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" @@ -2061,7 +2066,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), "python/loaders/orders.py": ( "from python.tasks.orders import prepare_orders\n" @@ -2073,7 +2078,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo " depends_on=(prepare_orders,),\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1}]\n" ), }, ) @@ -2124,21 +2129,21 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/orders.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), "python/loaders/orders.py": ( "from python.tasks.orders import prepare_orders\n" @@ -2150,7 +2155,7 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou " depends_on=(prepare_orders,),\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1}]\n" ), }, ) @@ -2204,13 +2209,13 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "from sqlbuild.checks import check\n\n" "@check(depends_on=prepare_orders)\n" "def check_orders_export(ctx):\n" - " return ctx.pass_()\n" + " '''Test check check_orders_export.'''\n return ctx.pass_()\n" ), "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py index ff7086fcfd..eb609edd8d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -22,7 +22,7 @@ ) _EVENTS_SEED_YML: str = ( "seeds:\n" - "- name: events\n" + "- name: events\n description: Test seed events.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -31,12 +31,12 @@ ) _EVENTS_CSV: str = "id,event_date\n1,2026-09-01\n2,2026-09-02\n3,2026-09-03\n" _UPSTREAM_SQL: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n id,\n CAST(event_date AS DATE) AS event_date\n" 'FROM __seed("events")\n' ) _DOWNSTREAM_MB_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor event_date,\n" @@ -59,7 +59,7 @@ 'SELECT\n id,\n event_date\nFROM __ref("upstream_v")\n' ) _DOWNSTREAM_MERGE_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -80,7 +80,7 @@ def _upstream_view_sql(*, upstream: str) -> str: return ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model udf__adjust_amount.',\n materialized view,\n);\n\n" f'SELECT\n id,\n event_date\nFROM __ref("{upstream}")\n' ) @@ -125,7 +125,7 @@ def insert_table_between_upstream_and_view(*, project_dir: Path) -> None: """Add a new table between the upstream table and its view.""" (project_dir / "models/upstream_new.sql").write_text( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model upstream_new.',\n materialized table,\n);\n\n" 'SELECT\n id,\n event_date\nFROM __ref("upstream")\n', encoding="utf-8", ) @@ -171,7 +171,7 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: "sqlbuild_project.toml": _PROJECT_TOML, "seeds/seeds.yml": ( "seeds:\n" - "- name: orders\n" + "- name: orders\n description: Test seed orders.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -184,10 +184,11 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: "id,ordered_at,amount\n1,2026-09-01,10\n2,2026-09-02,20\n3,2026-09-03,30\n" ), _ADJUST_FUNCTION_PATH: ( - "FUNCTION (\n arguments (amount INTEGER),\n returns INTEGER,\n);\n\namount + 1\n" + "FUNCTION (description 'Test function.'," + "\n arguments (amount INTEGER),\n returns INTEGER,\n);\n\namount + 1\n" ), "models/order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_amounts.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -202,7 +203,7 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: 'FROM __seed("orders")\n' ), "models/order_totals.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_totals.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -233,7 +234,7 @@ def direct_reference_merge_files() -> dict[str, str]: return { "models/down_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -270,11 +271,11 @@ def build_star_project(*, tmp_path: Path, downstream_config: str) -> Path: repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/up.sql": ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model up.',\n materialized table,\n);\n\n" "SELECT 1 AS id, DATE '2026-09-01' AS event_date\n" ), "models/down.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down.',\n" " materialized incremental,\n" " incremental_strategy append,\n" " replay_on_change full,\n" @@ -332,10 +333,11 @@ def udf_reference_merge_files() -> dict[str, str]: return { "functions/sql/udf__plus_one.sql": ( - "FUNCTION (\n arguments (value INTEGER),\n returns INTEGER,\n);\n\nvalue + 1\n" + "FUNCTION (description 'Test function udf__plus_one.'," + "\n arguments (value INTEGER),\n returns INTEGER,\n);\n\nvalue + 1\n" ), "models/down_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -359,7 +361,7 @@ def incremental_upstream_files() -> dict[str, str]: return { "models/upstream.sql": ( - "MODEL (\n" + "MODEL (description 'Test model upstream.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -397,7 +399,8 @@ def drop_upstream_column(*, project_dir: Path) -> None: """Drop the event_date column from the star project's upstream table.""" (project_dir / "models/up.sql").write_text( - "MODEL (\n materialized table,\n);\n\nSELECT 1 AS id\n", encoding="utf-8" + "MODEL (description 'Test model up.',\n materialized table,\n);\n\nSELECT 1 AS id\n", + encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py index eca6f20f34..a41b2e1e6d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py @@ -53,6 +53,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -60,7 +61,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, cursor ordered_at, @@ -180,6 +181,7 @@ def test_given_append_cursor_project_when_rerunning_build_then_boundary_behavior """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -187,7 +189,7 @@ def test_given_append_cursor_project_when_rerunning_build_then_boundary_behavior + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, cursor ordered_at, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py index e3de912241..259e54c649 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py @@ -41,7 +41,7 @@ def test_given_failing_factory_audit_when_building_then_existing_table_is_untouc materialized = "table" """, "models/orders.sql": """ -MODEL (audit_factories [order_quality]); +MODEL (description "Test model orders.", audit_factories [order_quality]); SELECT 2 AS id, -10 AS amount """, "audits/generic/expression_is_true.sql": """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py index ae4e460f6e..6a8674fac6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py @@ -61,7 +61,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy merge, unique_key [id], @@ -135,7 +135,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy merge, unique_key [id], @@ -210,7 +210,7 @@ def test_given_audit_failure_projects_when_running_build_then_cli_reports_failur '[connection]\ndatabase = "measurement.duckdb"\n' ), "models/orders.sql": ( - "MODEL (materialized table, audits [row_rate (" + "MODEL (description 'Test model orders.', materialized table, audits [row_rate (" "thresholds (error (below 90)))]) ; SELECT 1 AS order_id" ), "audits/generic/row_rate.sql": ( @@ -229,8 +229,12 @@ def test_given_audit_failure_projects_when_running_build_then_cli_reports_failur 'name = "measurement_audit"\nadapter = "duckdb"\n' '[connection]\ndatabase = "measurement.duckdb"\n' ), - "models/orders.sql": ("MODEL (materialized table); SELECT 1 AS order_id"), - "models/customers.sql": ("MODEL (materialized table); SELECT 1 AS order_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table); SELECT 1 AS order_id" + ), + "models/customers.sql": ( + "MODEL (description 'Test model customers.', materialized table); SELECT 1 AS order_id" + ), "audits/singular/rate.sql": ( "AUDIT (evaluation measurement, value rate, " "thresholds (error (below 90))); " diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 9ca3a1bad5..0e80b749db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -44,13 +44,13 @@ description="selected downstream clones missing upstream boundary from prod", project_name="defer_clone_build", initial_upstream_sql=( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'prod_version' AS label\n" + "MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id, 'prod_version' AS label\n" ), changed_upstream_sql=( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'dev_version' AS label\n" + "MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id, 'dev_version' AS label\n" ), downstream_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT id, label || '_downstream' AS label FROM __ref(\"upstream\")\n" ), prod_build_command=("--no-color", "build", "--target", "prod"), @@ -132,10 +132,10 @@ def test_given_changed_unselected_upstream_when_building_leaf_then_warns_and_noo "[connection]\n" f'database = "{test_case.project_name}.duckdb"\n' ), - "models/a.sql": "MODEL (materialized table);\n\nselect 1 as id\n", - "models/b.sql": "MODEL (materialized table);\n\nselect 1 as id\n", + "models/a.sql": "MODEL (description 'Test model a.', materialized table);\n\nselect 1 as id\n", + "models/b.sql": "MODEL (description 'Test model b.', materialized table);\n\nselect 1 as id\n", "models/c.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model c.', materialized table);\n\n" 'select * from __ref("a") union all select * from __ref("b")\n' ), }, @@ -146,11 +146,11 @@ def test_given_changed_unselected_upstream_when_building_leaf_then_warns_and_noo project_dir=project_dir, ) (project_dir / "models" / "a.sql").write_text( - "MODEL (materialized table);\n\nselect 10 as id\n", + "MODEL (description 'Test model a.', materialized table);\n\nselect 10 as id\n", encoding="utf-8", ) (project_dir / "models" / "b.sql").write_text( - "MODEL (materialized table);\n\nselect 2 as id\n", + "MODEL (description 'Test model b.', materialized table);\n\nselect 2 as id\n", encoding="utf-8", ) mixed_result: subprocess.CompletedProcess[str] = run_sqb( @@ -208,10 +208,10 @@ def test_given_direct_function_dependency_when_building_then_persists_function_h 'database = "warehouse.duckdb"\n' ), "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 100\n" + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 100\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT __udf("is_large_order")(150) AS is_large\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT __udf("is_large_order")(150) AS is_large\n' ), }, ) @@ -269,7 +269,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -277,7 +277,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist " query: SELECT data_version FROM freshness_control\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -298,7 +298,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist ) (project_dir / "models" / "orders.sql").write_text( ( - "MODEL (materialized table, " + "MODEL (description 'Test model orders.', materialized table, " 'post_hooks [inline_sql("UPDATE freshness_control SET data_version = 2")]);\n\n' 'SELECT * FROM __source("raw_orders")\n' ), @@ -359,13 +359,13 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 1 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" " freshness:\n" " strategy: sql\n" @@ -373,10 +373,10 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ " query: SELECT 1 AS data_version\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/payments.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_payments")\n' + 'MODEL (description "Test model payments.", materialized table);\n\nSELECT * FROM __source("raw_payments")\n' ), }, ) @@ -396,13 +396,13 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ ) (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 2 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" " freshness:\n" " strategy: sql\n" @@ -411,7 +411,7 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ encoding="utf-8", ) (project_dir / "models" / "payments.sql").write_text( - "MODEL (materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS payment_id\n", + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS payment_id\n", encoding="utf-8", ) execution_json_path: Path = project_dir / "target" / "failed-build.json" @@ -486,13 +486,13 @@ def test_given_source_freshness_error_when_building_then_blocks_only_affected_br customer_freshness_query="SELECT CURRENT_TIMESTAMP AS data_version", ), "models/stg_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized view);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "models/dim_customers.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_customers")\n' + 'MODEL (description "Test model dim_customers.", materialized table);\n\nSELECT * FROM __source("raw_customers")\n' ), }, ) @@ -566,13 +566,13 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 1 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id, 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -580,7 +580,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " query: SELECT 1 AS data_version\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT o.order_id, p.payment_id FROM __source("raw_orders") o ' 'JOIN __source("raw_payments") p USING (order_id)\n' ), @@ -602,13 +602,13 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ) (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 2 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id, 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -617,7 +617,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap encoding="utf-8", ) (project_dir / "models" / "fact_orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS order_id\n", + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS order_id\n", encoding="utf-8", ) @@ -660,18 +660,18 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " raise RuntimeError('prepare failed')\n" + " '''Test task prepare_orders.'''\n raise RuntimeError('prepare failed')\n" ), "python/loaders/raw.py": ( "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -679,7 +679,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, expected_exit_code=1, @@ -707,18 +707,18 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" + " '''Test task prepare_orders.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), "python/loaders/raw.py": ( "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -726,7 +726,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, expected_exit_code=0, @@ -753,7 +753,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_result(ctx):\n" - " return ctx.result(payload={'bad': {'a', 'b'}})\n" + " '''Test task produce_bad_result.'''\n return ctx.result(payload={'bad': {'a', 'b'}})\n" ), }, expected_exit_code=1, @@ -777,7 +777,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_metadata(ctx):\n" - " return ctx.result(payload={}, metadata={'bad': {'a', 'b'}})\n" + " '''Test task produce_bad_metadata.'''\n return ctx.result(payload={}, metadata={'bad': {'a', 'b'}})\n" ), }, expected_exit_code=1, @@ -799,21 +799,21 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def fail_after_fact(ctx):\n" - " raise RuntimeError('profile failed')\n" + " '''Test task fail_after_fact.'''\n raise RuntimeError('profile failed')\n" ), }, expected_exit_code=1, @@ -838,14 +838,14 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" @@ -853,7 +853,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def skip_after_fact(ctx):\n" - " return ctx.skip(reason='no profile needed', mode=SkipMode.HARD)\n" + " '''Test task skip_after_fact.'''\n return ctx.skip(reason='no profile needed', mode=SkipMode.HARD)\n" ), }, expected_exit_code=0, @@ -878,14 +878,14 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" @@ -893,7 +893,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def soft_skip_after_fact(ctx):\n" - " return ctx.skip(reason='optional profile skipped', mode=SkipMode.SOFT)\n" + " '''Test task soft_skip_after_fact.'''\n return ctx.skip(reason='optional profile skipped', mode=SkipMode.SOFT)\n" ), }, expected_exit_code=0, @@ -921,7 +921,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test task prepare_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result(payload={'order_id': 3})\n" ), @@ -930,11 +930,11 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 3}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 3}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -942,7 +942,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from pathlib import Path\n" @@ -950,7 +950,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('profile.txt')\n" + " '''Test task profile_fact.'''\n marker = Path(__file__).parents[2].joinpath('profile.txt')\n" " marker.write_text('profiled')\n" " return ctx.result()\n" ), @@ -979,11 +979,11 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 5}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 5}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -996,7 +996,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" - " relation = ctx.relation(source('raw_orders'))\n" + " '''Test task profile_raw_orders.'''\n relation = ctx.relation(source('raw_orders'))\n" " value = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" " marker = Path(__file__).parents[2].joinpath('source_profile.txt')\n" " marker.write_text(str(value))\n" @@ -1069,14 +1069,14 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, expected_exit_code=1, expected_output_fragments=("requires intermediate loader 'fetch_events'",), @@ -1097,7 +1097,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test task prepare_events.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result()\n" ), @@ -1106,11 +1106,11 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "from python.tasks.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1175,14 +1175,14 @@ def test_given_direct_source_only_build_when_loader_has_ingress_then_enforces_so " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, expected_exit_code=0, expected_output_fragments=("source raw_events",), @@ -1240,7 +1240,7 @@ def test_given_existing_intermediate_target_when_building_source_only_then_reuse "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" - " return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" + " '''Test asset observed_orders.'''\n return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" ), }, expected_exit_code=0, @@ -1348,14 +1348,14 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 7}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 7}]\n" ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" - " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" + " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), "python/assets/prepare.py": ( @@ -1364,7 +1364,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" - " payload = ctx.result_of(node_function=prepare_orders).payload\n" + " '''Test asset publish_prepared_orders.'''\n payload = ctx.result_of(node_function=prepare_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1375,12 +1375,12 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1388,14 +1388,14 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" + " '''Test task profile_fact_orders.'''\n relation = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), @@ -1404,7 +1404,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" + " '''Test asset export_fact_orders.'''\n payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), "python/tasks/notify.py": ( @@ -1413,7 +1413,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=export_fact_orders).payload\n" + " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" @@ -1500,7 +1500,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'first'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'first'})\n" ), }, ) @@ -1519,7 +1519,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 84}, metadata={'source': 'second'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 84}, metadata={'source': 'second'})\n" ), encoding="utf-8", ) @@ -1532,7 +1532,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " raise RuntimeError('producer failed after previous success')\n" + " '''Test task produce_result.'''\n raise RuntimeError('producer failed after previous success')\n" ), encoding="utf-8", ) @@ -1555,7 +1555,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" - " latest = ctx.result_of(node_function=produce_result)\n" + " '''Test task consume_result.'''\n latest = ctx.result_of(node_function=produce_result)\n" f" first = ctx.result_of(node_function=produce_result, run_id='{first_run_id}')\n" f" failed = ctx.result_of(node_function=produce_result, run_id='{failed_run_id}')\n" " history = ctx.results_of(node_function=produce_result, limit=2)\n" @@ -1643,11 +1643,11 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1655,7 +1655,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), "python/tasks/consume.py": ( "from pathlib import Path\n" @@ -1663,7 +1663,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_loader_result(ctx):\n" - " result = ctx.result_of(node_function=raw_events)\n" + " '''Test task consume_loader_result.'''\n result = ctx.result_of(node_function=raw_events)\n" " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" " output.write_text(\n" " f\"{result.metadata['loader_name']}:{result.metadata['source_name']}:\"\n" @@ -1712,11 +1712,11 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " raise RuntimeError('loader failed')\n" + " '''Test loader raw_events.'''\n raise RuntimeError('loader failed')\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1724,7 +1724,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), }, command=("--no-color", "build", "--select", "fact_events"), @@ -1746,18 +1746,18 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" + " '''Test task prepare_events.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), "python/loaders/events.py": ( "from python.tasks.prepare import prepare_events\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_events,))\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1765,7 +1765,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), }, command=("--no-color", "build", "--select", "+fact_events"), @@ -1845,7 +1845,7 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'target': ctx.target})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'target': ctx.target})\n" ), "python/tasks/consume.py": ( "from pathlib import Path\n" @@ -1853,7 +1853,7 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" - " result = ctx.result_of(node_function=produce_result)\n" + " '''Test task consume_result.'''\n result = ctx.result_of(node_function=produce_result)\n" " output = Path(__file__).parents[2].joinpath('target_result.txt')\n" " output.write_text(str(result.payload['target']))\n" " return ctx.result()\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py index 766986e419..d606fca874 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py @@ -55,11 +55,11 @@ def test_given_monthly_producer_with_midmonth_max_when_building_capped_daily_con ).strip() + "\n", "sources/raw.yml": ( - "sources:\n - name: raw_meetings\n schema: main\n table: raw_meetings\n" + "sources:\n - name: raw_meetings\n description: Test source raw_meetings.\n schema: main\n table: raw_meetings\n" ), "models/monthly_meetings.sql": dedent( """ - MODEL ( + MODEL (description "Test model monthly_meetings.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -80,7 +80,7 @@ def test_given_monthly_producer_with_midmonth_max_when_building_capped_daily_con + "\n", "models/daily_suffix.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_suffix.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -380,9 +380,11 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ """ sources: - name: raw_late + description: Test source raw_late. schema: main table: raw_late - name: raw_early + description: Test source raw_early. schema: main table: raw_early """ @@ -390,7 +392,7 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ + "\n", "models/capped_late.sql": dedent( """ - MODEL ( + MODEL (description "Test model capped_late.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -417,14 +419,14 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ + "\n", "models/uncapped_early.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model uncapped_early.", materialized table); SELECT id, event_time FROM __source("raw_early") """ ).strip() + "\n", "models/any_consumer.sql": dedent( """ - MODEL ( + MODEL (description "Test model any_consumer.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 2cfca30829..a55b5a8880 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -148,6 +148,7 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "relative_path": "hooks/sql/select_value.sql", "definition_sql": "SELECT @value", "kwargs": {"value": 1}, + "description": "Test hook select_value.", }, {"type": "python", "name": "notify", "kwargs": {"message": "starting"}}, ), @@ -176,11 +177,11 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m "from sqlbuild.hooks import hook\n\n" "@hook\n" "def notify(ctx, message):\n" - " ctx.log(message)\n" + " '''Test hook notify.'''\n ctx.log(message)\n" ), - "hooks/sql/select_value.sql": "HOOK ();\n\nSELECT @value\n", + "hooks/sql/select_value.sql": "HOOK (description 'Test hook select_value.');\n\nSELECT @value\n", "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" ' pre_hooks [sql("select_value", value: 1), ' 'python("notify", message: "starting")],\n' @@ -216,7 +217,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m "type": "sql", "name": "select_value", "relative_path": "hooks/sql/select_value.sql", - "description": None, + "description": "Test hook select_value.", "sql": "SELECT @value", } assert hooks_by_name["notify"]["relative_path"] == "hooks/python/notify.py" @@ -231,7 +232,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m description="unknown materialization stops compile", repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (materialized tablex);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized tablex);\n\nSELECT 1 AS order_id\n", }, expected_code="P001", expected_message_fragment="unknown materialization 'tablex'", @@ -242,7 +243,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + "MODEL (description 'Test model orders.',\n materialized incremental,\n incremental_strategy append,\n" " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py index 9510a9d06b..be98fffbb3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py @@ -51,7 +51,7 @@ def test_given_project_capability_disabled_when_batch_concurrency_exceeds_one_th + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -73,7 +73,7 @@ def test_given_project_capability_disabled_when_batch_concurrency_exceeds_one_th ).strip() + "\n", "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n schema: main\n table: raw_events\n" ), }, ) @@ -120,6 +120,7 @@ def test_given_opt_in_concurrent_microbatch_when_building_twice_then_records_all """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -127,7 +128,7 @@ def test_given_opt_in_concurrent_microbatch_when_building_twice_then_records_all + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -245,6 +246,7 @@ def test_given_later_batches_succeed_when_one_batch_fails_then_next_run_preserve """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -252,7 +254,7 @@ def test_given_later_batches_succeed_when_one_batch_fails_then_next_run_preserve + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -402,6 +404,7 @@ def test_given_microbatch_history_is_lost_when_reconciling_then_policy_is_applie """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -409,7 +412,7 @@ def test_given_microbatch_history_is_lost_when_reconciling_then_policy_is_applie + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -511,6 +514,7 @@ def test_given_bounded_replay_on_change_when_model_changes_then_requirement_is_d """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -610,6 +614,7 @@ def test_given_full_replay_fails_when_retried_then_original_requirement_is_reuse """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -700,6 +705,7 @@ def test_given_version_hash_returns_when_replaying_then_new_transition_gets_new_ """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -790,6 +796,7 @@ def test_given_historical_backfill_when_normal_run_resumes_then_gap_to_normal_fl """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -797,7 +804,7 @@ def test_given_historical_backfill_when_normal_run_resumes_then_gap_to_normal_fl + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -898,6 +905,7 @@ def test_given_no_new_cursor_work_when_history_is_lost_then_reconciliation_still """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -905,7 +913,7 @@ def test_given_no_new_cursor_work_when_history_is_lost_then_reconciliation_still + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py index 9ac5975d75..559fae9c2b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py @@ -36,11 +36,11 @@ "(customer_id INTEGER, last_ordered_at TIMESTAMP);\n" ), "models/stg_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_orders.', materialized view);\n\n" "SELECT id AS order_id, ordered_at FROM main.raw_orders\n" ), "models/fact_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model fact_orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (\n" @@ -52,7 +52,7 @@ ), "models/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -94,7 +94,7 @@ ), "models/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [customer_id], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py index fefccc6727..aa73f3aca0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py @@ -48,7 +48,7 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t '"SQBRCONTRACT106"]\n' ), "models/stg_order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_amounts.',\n" " materialized table,\n" " columns (\n" " customer_id (type INTEGER),\n" @@ -67,7 +67,7 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t "CAST(7.00 AS DECIMAL(12,2))\n" ), "models/customer_category_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " materialized table,\n" " columns (customer_id (type INTEGER)),\n" " dynamic_columns (\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py index edc5ff6bb1..3a3bd5c27d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py @@ -47,7 +47,7 @@ def test_given_enum_typed_contract_when_value_is_invalid_then_build_fails_audit( "models/_enums/fulfillment_method.sql": "ENUM (name fulfillment_method, members [DELIVERY, PICKUP]);\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized table, contract enforced, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py index 1889731764..9af51c2bcc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py @@ -51,8 +51,8 @@ def test_given_target_model_limit_when_building_then_effective_policy_controls_e max_models = 1 remediation = "Automated tools must request approval before changing this policy." """.strip(), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "models/order_items.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id\n", + "models/order_items.sql": "MODEL (description 'Test model order_items.', materialized table);\n\nSELECT 1 AS id\n", }, ) @@ -112,8 +112,8 @@ def test_given_local_model_limit_override_when_building_then_local_policy_is_app [targets.dev.execution_limits] max_models = 2 """.strip(), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "models/order_items.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id\n", + "models/order_items.sql": "MODEL (description 'Test model order_items.', materialized table);\n\nSELECT 1 AS id\n", }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py index a8fd48689f..320f93323e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py @@ -40,7 +40,7 @@ EXPLICIT_REFERENCE_MACROS + "\n\ndef orders_base():\n return 'SELECT * FROM __ref(\"stg_orders_eu\")'\n" ) -_EMITTING_MODEL: str = "MODEL (materialized table);\n@orders_base()\n" +_EMITTING_MODEL: str = "MODEL (description 'Test model.', materialized table);\n@orders_base()\n" _SECOND_EMITTING_MACROS: str = ( "def customers_base():\n return 'SELECT * FROM __ref(\"stg_customers\")'\n" ) @@ -50,8 +50,10 @@ "AUDIT ();\nSELECT s.order_id FROM @relation s\n" 'LEFT JOIN __ref("{read}") a USING (order_id) WHERE a.order_id IS NULL\n' ) -_AUDITED_MODEL: str = "MODEL (materialized table, audits [{audit}]);\nSELECT 1 AS order_id\n" -_SUMMARY_MODEL: str = 'MODEL (materialized table);\nSELECT * FROM __ref("{read}")\n' +_AUDITED_MODEL: str = "MODEL (description 'Test model.', materialized table, audits [{audit}]);\nSELECT 1 AS order_id\n" +_SUMMARY_MODEL: str = ( + 'MODEL (description "Test model.", materialized table);\nSELECT * FROM __ref("{read}")\n' +) _ONE_MODEL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("{read}") WHERE order_id IS NULL\n' _NULL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL\n' _LITERAL_TASK: str = ( @@ -59,14 +61,14 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" - ' ctx.query("SELECT count(*) FROM stg_customers")\n' + ' """Test task export_orders."""\n ctx.query("SELECT count(*) FROM stg_customers")\n' ) _RUNTIME_TASK: str = ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" - ' orders = ctx.relation(model("all_orders"))\n' + ' """Test task export_orders."""\n orders = ctx.relation(model("all_orders"))\n' ' ctx.query(f"SELECT count(*) FROM {orders}")\n' ' customers = "stg_" + "customers"\n' ' ctx.query(f"SELECT count(*) FROM {customers}")\n' @@ -75,7 +77,7 @@ _COUNTRY_SEED_FILES: dict[str, str] = { "seeds/countries.yml": ( - "seeds:\n - name: countries\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: countries\n description: Test seed countries.\n columns:\n - name: code\n type: VARCHAR\n" ), "seeds/countries.csv": "code\nGB\nFR\n", } @@ -84,7 +86,7 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=seed("countries"))\n' "def count_countries(ctx):\n" - ' countries = ctx.relation(seed("countries"))\n' + ' """Test task count_countries."""\n countries = ctx.relation(seed("countries"))\n' " ctx.execute_sql(\n" ' f"CREATE OR REPLACE TABLE country_counts AS SELECT count(*) AS n FROM {countries}"\n' " )\n" @@ -94,7 +96,7 @@ "from sqlbuild.refs import model\n\n\n" '@check(depends_on=model("all_orders"))\n' "def orders_present(ctx):\n" - ' orders = ctx.relation(model("all_orders"))\n' + ' """Test check orders_present."""\n orders = ctx.relation(model("all_orders"))\n' ' count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]\n' ' return ctx.pass_() if count else ctx.fail(message="no orders")\n' ) @@ -109,9 +111,9 @@ ) _RAW_SOURCES: str = ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" ) @@ -124,10 +126,10 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_summary(ctx):\n" - ' ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n' + ' """Test task export_summary."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n' '@task(depends_on=model("stg_orders_eu"))\n' "def export_eu(ctx):\n" - ' ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n' + ' """Test task export_eu."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n' ) @@ -205,7 +207,7 @@ def test_given_python_seed_and_model_dependencies_when_building_then_graph_selec UnrenderedMacroArgumentE2ETestCase( description="unchosen dictionary entry is a dependency the build selects", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" '@pick({"eu": __ref("stg_orders_eu"), "us": __ref("stg_orders_us")}, key="eu")\n' ), expected_dag_edges=( @@ -385,7 +387,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an expected_exit_code=1, expected_output_fragments=( "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'", - "--> python/tasks/export.py:7", + "--> python/tasks/export.py:8", "[references]\n enforce_explicit = false", ), ), @@ -396,7 +398,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an expected_exit_code=1, expected_output_fragments=( "error[P008]: check:orders_present names model:stg_customers as 'stg_customers'", - "--> python/checks/orders.py:8", + "--> python/checks/orders.py:9", 'declare it with depends_on=model("stg_customers")', ), ), @@ -570,7 +572,7 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit "models/sales/_sqlbuild/_macros/customers.py": _SECOND_EMITTING_MACROS, f"{EXPLICIT_REFERENCE_MODELS}/hidden.sql": _EMITTING_MODEL, f"{EXPLICIT_REFERENCE_MODELS}/hidden_customers.sql": ( - "MODEL (materialized table);\n@customers_base()\n" + "MODEL (description 'Test model hidden_customers.', materialized table);\n@customers_base()\n" ), "python/tasks/export.py": _LITERAL_TASK, }, @@ -649,11 +651,11 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit "audits/generic/us_check.sql": _NULL_AUDIT, f"{_AUDITS}/audits/generic/unused_check.sql": _NULL_AUDIT, f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": ( - "MODEL (materialized table, audits [eu_check]);\n" + "MODEL (description 'Test model stg_orders_eu.', materialized table, audits [eu_check]);\n" "SELECT 1 AS order_id, 10 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": ( - "MODEL (materialized table, audits [us_check]);\n" + "MODEL (description 'Test model stg_orders_us.', materialized table, audits [us_check]);\n" "SELECT 2 AS order_id, 20 AS amount\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py index d49a3ff666..99addcdb95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py @@ -42,6 +42,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, '1', '2026-01-01 00:30:00', 100, 'web'), @@ -64,7 +65,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py index d0b28ade69..a21a982ccb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py @@ -57,13 +57,13 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " maximum = int(Path(__file__).parents[2].joinpath('maximum.txt').read_text())\n" + " '''Test loader raw_orders.'''\n maximum = int(Path(__file__).parents[2].joinpath('maximum.txt').read_text())\n" " return [{'id': value, 'amount': value * 100} " "for value in range(1, maximum + 1)]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -74,7 +74,7 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ ), "models/raw_orders_incremental.sql": dedent( """ - MODEL ( + MODEL (description "Test model raw_orders_incremental.", materialized incremental, incremental_strategy delete_insert, cursor id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py index b1803c99ca..3e6c5c4d3f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py @@ -40,7 +40,7 @@ def test_given_direct_project_when_building_then_manifest_requires_compile_manif "[connection]\n" 'database = "warehouse.duckdb"\n' ), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", }, ) manifest_path: Path = project_dir / "target" / "manifest.json" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py index 8af804249c..0fce894f98 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py @@ -31,6 +31,7 @@ """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events columns: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py index 0d71df9d74..5bd9329e48 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py @@ -132,7 +132,7 @@ def test_given_integer_cursor_maximum_when_building_then_half_open_batches_inclu ), "sources/raw.yml": raw_events_source_yml(), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py index b0ae8fb8ad..57fe340b2d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py @@ -71,6 +71,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -78,7 +79,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, @@ -91,7 +92,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, quantity, ordered_at, line_total_cents FROM __ref("stg_orders") @@ -100,7 +101,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -127,7 +128,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -154,7 +155,7 @@ + "\n", "models/marts/hourly_activity_with_daily_context.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_activity_with_daily_context.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py index b93ea51daf..fc0f0bbf94 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py @@ -59,6 +59,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -66,7 +67,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view ); @@ -83,7 +84,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table ); @@ -100,7 +101,7 @@ + "\n", "models/intermediate/order_status_index.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_status_index.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -124,7 +125,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -154,7 +155,7 @@ + "\n", "models/marts/fact_orders_view.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders_view.", materialized view ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py index de9a9eee00..588c072fbe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py @@ -28,7 +28,7 @@ "from sqlbuild.hooks import hook\n\n\n" "@hook\n" "def fail_hook(ctx, message):\n" - " raise RuntimeError(message)\n" + " '''Test hook fail_hook.'''\n raise RuntimeError(message)\n" ) @@ -74,13 +74,14 @@ def test_given_python_pre_hook_returns_skip_when_building_then_model_and_downstr @hook def skip_model(ctx): + '''Test hook skip_model.''' return ctx.skip(reason="source disabled") """ ).strip() + "\n", "models/upstream_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model upstream_orders.", materialized table, pre_hooks [python("skip_model")] ); @@ -91,7 +92,7 @@ def skip_model(ctx): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT order_id FROM __ref("upstream_orders") """ @@ -158,13 +159,14 @@ def test_given_python_post_hook_skip_when_building_then_keeps_relation_and_skips @hook def skip_downstream(ctx): + '''Test hook skip_downstream.''' return ctx.skip(reason="publish disabled") """ ).strip() + "\n", "models/upstream_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model upstream_orders.", materialized table, post_hooks [python("skip_downstream")] ); @@ -175,7 +177,7 @@ def skip_downstream(ctx): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT order_id FROM __ref("upstream_orders") """ @@ -245,6 +247,7 @@ def test_given_project_with_python_hooks_when_building_then_hooks_execute( @hook def create_hook_data(ctx, value): + '''Test hook create_hook_data.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.hook_data AS " f"SELECT {value} AS id, 'created by hook' AS label" @@ -253,6 +256,7 @@ def create_hook_data(ctx, value): @hook def record_hook_completion(ctx, phase): + '''Test hook record_hook_completion.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.hook_log AS " f"SELECT '{ctx.model_name}' AS model_name, " @@ -272,7 +276,7 @@ def record_hook_completion(ctx, phase): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [sql("select_one", value: 1), python("create_hook_data", value: 42)], post_hooks [python("record_hook_completion", phase: "post"), inline_sql("SELECT 1")] @@ -426,9 +430,11 @@ def test_given_python_hooks_lifecycle_matrix_when_building_then_all_materializat """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -441,6 +447,7 @@ def test_given_python_hooks_lifecycle_matrix_when_building_then_all_materializat @hook def log_hook(ctx): + '''Test hook log_hook.''' ctx.execute_sql( f"INSERT INTO {ctx.destination.schema}.hook_log VALUES " f"('{ctx.model_name}', '{ctx.phase}')" @@ -464,7 +471,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -483,7 +490,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -502,7 +509,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/intermediate/order_status_index.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_status_index.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -526,7 +533,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -555,7 +562,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/snapshots/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -571,7 +578,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/custom/custom_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model custom_orders.", materialized copy_table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -652,13 +659,14 @@ def test_given_python_post_hook_failure_when_building_graph_then_downstream_is_b @hook def fail_hook(ctx, message): + '''Test hook fail_hook.''' raise RuntimeError(message) """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [python("fail_hook", message: "intentional post failure")] ); @@ -669,7 +677,7 @@ def fail_hook(ctx, message): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT id FROM __ref("orders") """ @@ -700,7 +708,7 @@ def fail_hook(ctx, message): description="pre hook failure shows failing Python hook row", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model.", materialized table, pre_hooks [python("fail_hook", message: "intentional pre failure")] ); @@ -722,7 +730,7 @@ def fail_hook(ctx, message): description="pre hook failure shows failing SQL hook row", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model lifecycle.", materialized table, pre_hooks [inline_sql("SELECT * FROM missing_hook_table")] ); @@ -820,10 +828,10 @@ def test_given_long_python_hook_name_when_building_then_cli_truncates_label_at_c "@hook\n" "def publish_customer_metadata_to_external_catalog_after_successful_" "materialization(ctx):\n" - ' ctx.log("long hook ran")\n' + ' """Test hook publish_customer_metadata_to_external_catalog_after_successful_materialization."""\n ctx.log("long hook ran")\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" ' pre_hooks [python("publish_customer_metadata_to_external_catalog_after_' 'successful_materialization")]\n' @@ -882,6 +890,7 @@ def test_given_snapshot_with_python_hooks_when_building_then_hooks_execute( @hook def create_snapshot_source(ctx): + '''Test hook create_snapshot_source.''' ctx.execute_sql( "CREATE TABLE main.raw_customers AS " "SELECT 1 AS customer_id, 'basic' AS plan, " @@ -891,6 +900,7 @@ def create_snapshot_source(ctx): @hook def log_snapshot_hook(ctx): + '''Test hook log_snapshot_hook.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.snapshot_hook_log AS " f"SELECT '{ctx.model_name}' AS model_name, '{ctx.phase}' AS phase" @@ -900,7 +910,7 @@ def log_snapshot_hook(ctx): + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index a2cc997b96..8d595bfc27 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -77,11 +77,13 @@ def test_given_built_python_nodes_when_editing_decorator_inputs_then_identity_tr @loader(write_strategy="table", columns=STAGED_ORDER_COLUMNS) def staged_orders(ctx): + '''Test loader staged_orders.''' return [{{"order_id": 1}}] @loader(depends_on=[staged_orders]) def raw_orders(ctx): + '''Test loader raw_orders.''' staged = ctx.loader(staged_orders).destination rows = ctx.query(f"SELECT order_id FROM {{staged}}") return [{{"order_id": row[0]}} for row in rows] @@ -91,6 +93,7 @@ def raw_orders(ctx): """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -108,6 +111,7 @@ def raw_orders(ctx): @asset(retry=EXPORT_RETRY) def export_orders(ctx): + '''Test asset export_orders.''' return None """ ).lstrip(), @@ -197,6 +201,7 @@ def normalize_status(value): @task def orders(ctx): + '''Test task orders.''' return ctx.result( payload={"status": python.helpers.clean.normalize_status(" Shipped ")} ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 2dbec73617..c792b221af 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -24,7 +24,7 @@ ) _FAILING_ORDERS_SQL: str = ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" "SELECT order_id FROM __source(\"raw_orders\") WHERE error('orders failed') IS NULL\n" ) _EXISTING_ORDERS: str = "CREATE TABLE main.orders AS SELECT * FROM range(1, 6) AS t(order_id);" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py index 5db2a5626a..244dbfa4ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py @@ -28,33 +28,33 @@ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 9}]\n" ), "models/sales/_sqlbuild/_hooks/python/count.py": ( "from sqlbuild.hooks import hook\nfrom sqlbuild.refs import source\n\n\n" '@hook(reads=source("raw_orders"))\n' "def count_raw(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test hook count_raw."""\n raw = ctx.relation(source("raw_orders"))\n' ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.hook_counts AS ' 'SELECT count(*) AS n FROM {raw}")\n' ), "models/sales/order_count.sql": ( - 'MODEL (materialized table, post_hooks [python("count_raw")]);\n' + 'MODEL (description "Test model order_count.", materialized table, post_hooks [python("count_raw")]);\n' 'SELECT count(*) AS n FROM __source("raw_orders")\n' ), "models/sales/order_marker.sql": ( - 'MODEL (materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n' + 'MODEL (description "Test model order_marker.", materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n' ), "python/tasks/count.py": ( "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" '@task(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test task count_task."""\n raw = ctx.relation(source("raw_orders"))\n' ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.task_counts AS ' 'SELECT count(*) AS n FROM {raw}")\n' ), @@ -62,7 +62,7 @@ "from sqlbuild.checks import check\nfrom sqlbuild.refs import source\n\n\n" '@check(depends_on=source("raw_orders"))\n' "def two_raw_orders(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test check two_raw_orders."""\n raw = ctx.relation(source("raw_orders"))\n' ' n = ctx.query(f"SELECT count(*) FROM {raw}").fetchone()[0]\n' ' return ctx.pass_() if n == 2 else ctx.fail(message=f"read {n} rows")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 3a730bbdae..4fe9306891 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -58,6 +58,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -65,7 +66,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view ); @@ -80,7 +81,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py index 1453ee5c17..5be91ae74d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py @@ -26,18 +26,18 @@ '[defaults]\nmaterialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount_cents INTEGER), returns BOOLEAN);\n\namount_cents > 100\n" + "FUNCTION (description 'Test function is_large_order.', arguments (amount_cents INTEGER), returns BOOLEAN);\n\namount_cents > 100\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "SELECT id AS order_id, amount_cents, 'web order' AS channel\n" 'FROM __source("raw_orders")\n' ), "models/large_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model large_orders.', materialized table);\n\n" "SELECT order_id, is_large_order(amount_cents) AS is_large\n" 'FROM __ref("stg_orders")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py index f803825f7b..a8136a594a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py @@ -56,6 +56,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -63,7 +64,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, ordered_at, amount_cents FROM __source("raw_orders") """ @@ -71,7 +72,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at, amount_cents AS line_total_cents FROM __ref("stg_orders") @@ -80,7 +81,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -106,7 +107,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -179,6 +180,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -186,7 +188,7 @@ + "\n", "functions/sql/is_high_value_order.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function is_high_value_order.", arguments (amount_cents INTEGER), returns BOOLEAN, ); @@ -197,7 +199,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -262,6 +264,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -269,7 +272,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, ordered_at, amount_cents FROM __source("raw_orders") """ @@ -277,7 +280,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at, amount_cents AS line_total_cents FROM __ref("stg_orders") @@ -286,7 +289,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -313,7 +316,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py index f224bc1f82..741056fb97 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py @@ -41,14 +41,15 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ ).lstrip(), - "models/order_regions.sql": "MODEL (materialized table);\n\nSELECT 1 AS region_id\n", + "models/order_regions.sql": "MODEL (description 'Test model order_regions.', materialized table);\n\nSELECT 1 AS region_id\n", "models/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -66,7 +67,7 @@ _DAILY_ORDERS_CONFIG: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -99,7 +100,7 @@ _SOURCE_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -118,7 +119,7 @@ _MICROBATCH_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -145,7 +146,7 @@ _WINDOWED_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py index a9a15bb666..16e872ba43 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py @@ -47,6 +47,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -54,7 +55,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization + "\n", "schemas/events/base.sql": dedent( """ - SCHEMA ( + SCHEMA (description "Test schema.", name event, columns ( event_id (type INTEGER, nullable false, audits [not_null]), @@ -66,7 +67,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization + "\n", "schemas/events/sourced.sql": dedent( """ - SCHEMA ( + SCHEMA (description "Test schema.", name sourced_event, extends event, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py index 26ff549b22..3f23f0de51 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py @@ -40,7 +40,7 @@ "models/marts/daily/orders.sql": _MART_MODEL, "models/marts/weekly/orders_weekly.sql": _MART_MODEL, "models/marts/_sqlbuild/schemas/order_shape.sql": ( - "SCHEMA (name order_shape, columns (order_id (type INTEGER, nullable false)));\n" + "SCHEMA (description 'Test schema.', name order_shape, columns (order_id (type INTEGER, nullable false)));\n" ), "models/marts/_sqlbuild/audits/generic/order_check.sql": ( 'AUDIT ();\n\nSELECT order_id FROM __ref("@model") WHERE order_id IS NULL\n' @@ -52,18 +52,18 @@ "WHERE w.order_id IS NULL\n" ), "models/marts/_sqlbuild/hooks/sql/record_start.sql": ( - "HOOK ();\n\nINSERT INTO main.hook_log VALUES ('sql')\n" + "HOOK (description 'Test hook record_start.');\n\nINSERT INTO main.hook_log VALUES ('sql')\n" ), "models/marts/_sqlbuild/hooks/python/lifecycle.py": ( "from sqlbuild.hooks import hook\n\n\n" "@hook\n" "def record_finish(ctx):\n" - " ctx.execute_sql(\"INSERT INTO main.hook_log VALUES ('python')\")\n" + " '''Test hook record_finish.'''\n ctx.execute_sql(\"INSERT INTO main.hook_log VALUES ('python')\")\n" ), } _SEED_YAML: str = ( "seeds:\n" - " - name: order_codes\n" + " - name: order_codes\n description: Test seed order_codes.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -192,7 +192,7 @@ def test_given_seed_audit_when_building_then_it_runs_after_load_and_gates_depend "seeds/order_codes.yml": _SEED_YAML, "seeds/order_codes.csv": test_case.seed_csv, "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, label FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", materialized table);\n\nSELECT order_id, label FROM __seed("order_codes")\n' ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py index 0ff6254fa2..871ed4c383 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py @@ -54,7 +54,7 @@ def test_given_seed_watermark_when_seed_changes_then_incremental_consumes_new_hi + "\n", "seeds/schema.yml": ( "seeds:\n" - " - name: order_events\n" + " - name: order_events\n description: Test seed order_events.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -64,7 +64,7 @@ def test_given_seed_watermark_when_seed_changes_then_incremental_consumes_new_hi "seeds/order_events.csv": test_case.initial_seed, "models/order_events_incremental.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_events_incremental.", materialized incremental, incremental_strategy delete_insert, cursor id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py index b1a0c16b71..cac6c592c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py @@ -103,6 +103,7 @@ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -110,7 +111,7 @@ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -124,7 +125,7 @@ + "\n", "models/current_customer_plans.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model current_customer_plans.", materialized table); SELECT customer_id, plan FROM __ref("customer_snapshot") @@ -289,6 +290,7 @@ def test_given_snapshot_selector_when_excluding_downstream_then_only_snapshot_bu """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -296,7 +298,7 @@ def test_given_snapshot_selector_when_excluding_downstream_then_only_snapshot_bu + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -389,15 +391,19 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va """ sources: - name: raw_current_customers + description: Test source raw_current_customers. schema: main table: raw_current_customers - name: raw_historical_customers + description: Test source raw_historical_customers. schema: main table: raw_historical_customers - name: raw_initial_customers + description: Test source raw_initial_customers. schema: main table: raw_initial_customers - name: raw_audit_customers + description: Test source raw_audit_customers. schema: main table: raw_audit_customers """ @@ -405,7 +411,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/current_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model current_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -420,7 +426,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/historical_out_of_order_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model historical_out_of_order_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -436,7 +442,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/initial_updated_at_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model initial_updated_at_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -451,7 +457,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/audited_customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model audited_customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -672,6 +678,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -679,7 +686,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -745,6 +752,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -752,7 +760,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -824,6 +832,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -831,7 +840,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -918,6 +927,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -925,7 +935,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1092,15 +1102,19 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( """ sources: - name: raw_current_timestamp_customers + description: Test source raw_current_timestamp_customers. schema: main table: raw_current_timestamp_customers - name: raw_current_check_customers + description: Test source raw_current_check_customers. schema: main table: raw_current_check_customers - name: raw_historical_snapshot_customers + description: Test source raw_historical_snapshot_customers. schema: main table: raw_historical_snapshot_customers - name: raw_historical_change_customers + description: Test source raw_historical_change_customers. schema: main table: raw_historical_change_customers """ @@ -1108,7 +1122,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_timestamp_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_timestamp_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1122,7 +1136,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_status_check_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_check_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -1136,7 +1150,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_historical_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_historical_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1152,7 +1166,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_change_records_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_change_records_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1560,15 +1574,19 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns """ sources: - name: raw_customer_regions + description: Test source raw_customer_regions. schema: main table: raw_customer_regions - name: raw_customer_deletes + description: Test source raw_customer_deletes. schema: main table: raw_customer_deletes - name: raw_customer_status_history + description: Test source raw_customer_status_history. schema: main table: raw_customer_status_history - name: raw_customer_status_hard_delete_history + description: Test source raw_customer_status_hard_delete_history. schema: main table: raw_customer_status_hard_delete_history """ @@ -1576,7 +1594,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_region_custom_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_region_custom_snapshot.", materialized snapshot, unique_key [customer_id, region], snapshot_strategy timestamp, @@ -1592,7 +1610,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1607,7 +1625,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_status_historical_check_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_historical_check_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -1622,7 +1640,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_status_historical_check_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_historical_check_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2000,9 +2018,11 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t """ sources: - name: raw_membership_observations + description: Test source raw_membership_observations. schema: main table: raw_membership_observations - name: raw_membership_full_history + description: Test source raw_membership_full_history. schema: main table: raw_membership_full_history """ @@ -2010,7 +2030,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_incremental_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_incremental_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2026,7 +2046,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_full_history_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_full_history_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2111,9 +2131,11 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t """ sources: - name: raw_membership_observations + description: Test source raw_membership_observations. schema: main table: raw_membership_observations - name: raw_membership_full_history + description: Test source raw_membership_full_history. schema: main table: raw_membership_full_history """ @@ -2121,7 +2143,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_incremental_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_incremental_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2138,7 +2160,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_full_history_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_full_history_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2289,6 +2311,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2296,7 +2319,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2366,6 +2389,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2373,7 +2397,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2490,6 +2514,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2497,7 +2522,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2544,6 +2569,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2551,7 +2577,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2603,6 +2629,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2610,7 +2637,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2662,6 +2689,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2669,7 +2697,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2712,6 +2740,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2719,7 +2748,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2765,6 +2794,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2772,7 +2802,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2847,6 +2877,7 @@ def test_given_duplicate_timestamp_snapshot_source_when_building_then_cli_report """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2854,7 +2885,7 @@ def test_given_duplicate_timestamp_snapshot_source_when_building_then_cli_report + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2965,6 +2996,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2972,7 +3004,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3039,6 +3071,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3046,7 +3079,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3102,6 +3135,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3109,7 +3143,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3165,6 +3199,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3172,7 +3207,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3288,6 +3323,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3295,7 +3331,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3366,6 +3402,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3373,7 +3410,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3443,6 +3480,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3450,7 +3488,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3568,6 +3606,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -3575,7 +3614,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3647,6 +3686,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -3654,7 +3694,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3778,6 +3818,7 @@ def test_given_historical_timestamp_snapshot_project_when_rerunning_build_then_t """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -3785,7 +3826,7 @@ def test_given_historical_timestamp_snapshot_project_when_rerunning_build_then_t + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3904,6 +3945,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3911,7 +3953,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3954,6 +3996,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3961,7 +4004,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4036,6 +4079,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4043,7 +4087,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4088,6 +4132,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4095,7 +4140,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4184,6 +4229,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4191,7 +4237,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4249,6 +4295,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4256,7 +4303,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4310,6 +4357,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4317,7 +4365,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4388,6 +4436,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4395,7 +4444,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4466,6 +4515,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4473,7 +4523,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4550,6 +4600,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4557,7 +4608,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4633,6 +4684,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -4640,7 +4692,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4717,6 +4769,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -4724,7 +4777,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4800,6 +4853,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4807,7 +4861,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4884,6 +4938,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4891,7 +4946,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py index 86f73a6061..380a026955 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py @@ -77,7 +77,7 @@ def test_given_built_model_with_sql_hook_when_editing_hook_then_plans_expected_r _HOOK_PATH: _HOOK_CONTENTS, "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [sql("record_order_access", label: 1)] ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py index 6e005aff84..4f9231024e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py @@ -30,15 +30,15 @@ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 9}]\n" ), f"{_MODELS}/z_orders.sql": ( - "MODEL (materialized table);\nSELECT 1 AS order_id UNION ALL SELECT 2\n" + "MODEL (description 'Test model z_orders.', materialized table);\nSELECT 1 AS order_id UNION ALL SELECT 2\n" ), f"{_MODELS}/_sqlbuild/_macros/counts.py": ( "def count_orders(relation):\n" @@ -46,16 +46,16 @@ 'SELECT count(*) AS n FROM {relation}"\n' ), f"{_MODELS}/_sqlbuild/_hooks/sql/record_counts.sql": ( - 'HOOK ();\n@count_orders(__ref("z_orders"))\n' + 'HOOK (description "Test hook record_counts.");\n@count_orders(__ref("z_orders"))\n' ), f"{_MODELS}/a_report.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model a_report.',\n materialized table,\n" " pre_hooks [inline_sql('CREATE OR REPLACE TABLE main.inline_counts AS " 'SELECT count(*) AS n FROM __ref("z_orders")\')],\n' ' post_hooks [sql("record_counts")],\n);\nSELECT 1 AS n\n' ), f"{_MODELS}/raw_marker.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model raw_marker.',\n materialized table,\n" " post_hooks [inline_sql('CREATE OR REPLACE TABLE main.raw_counts AS " 'SELECT count(*) AS n FROM __source("raw_orders")\')],\n);\nSELECT 1 AS n\n' ), @@ -126,10 +126,10 @@ def test_given_sql_hook_references_when_building_then_they_resolve_and_order_lik description="inline hook reading a model built from its own model", overrides={ f"{_MODELS}/report_rollup.sql": ( - 'MODEL (materialized table);\nSELECT * FROM __ref("a_report")\n' + 'MODEL (description "Test model report_rollup.", materialized table);\nSELECT * FROM __ref("a_report")\n' ), f"{_MODELS}/a_report.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model a_report.',\n materialized table,\n" " post_hooks [inline_sql('SELECT * FROM __ref(\"report_rollup\")')],\n" ");\nSELECT 1 AS n\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py index ea592a4c7e..608578127b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py @@ -49,7 +49,7 @@ def test_given_model_table_function_dependency_when_building_then_creates_and_co + "\n", "functions/sql/customer_orders.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table ( customer_id INTEGER, @@ -63,7 +63,7 @@ def test_given_model_table_function_dependency_when_building_then_creates_and_co + "\n", "models/customer_order_summary.sql": dedent( f""" - MODEL ( + MODEL (description "Test model customer_order_summary.", materialized table ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py index f3245ea09d..f1d336126f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py @@ -66,6 +66,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -73,7 +74,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", cursor ordered_at, cursor_type timestamp, cursor_grain second, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py index 1e2db0e347..26f27dd4bf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py @@ -36,7 +36,7 @@ "seed_raw_data.sql": "", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [inline_sql('THIS IS NOT VALID SQL')] ); @@ -89,6 +89,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -96,7 +97,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, @@ -107,7 +108,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at FROM __ref("stg_orders") """ @@ -115,7 +116,7 @@ + "\n", "models/marts/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [order_id], @@ -157,13 +158,14 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: SELECT FROM """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model orders.", materialized table); SELECT * FROM __source("raw_orders") """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py index ce59cfcd15..0471c3810d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py @@ -29,11 +29,13 @@ def prepare_python_check_project(*, tmp_path: Path) -> Path: @task() def export_orders(ctx): + '''Test task export_orders.''' return ctx.result(payload={"rows": 3}, metadata={"rows": 3}) @task() def export_customers(ctx): + '''Test task export_customers.''' return ctx.result(payload={"rows": 2}, metadata={"rows": 2}) """, "python/assets/orders.py": """ @@ -43,6 +45,7 @@ def export_customers(ctx): @asset(depends_on=export_orders) def orders_asset(ctx): + '''Test asset orders_asset.''' return ctx.result(payload={"asset_rows": 3}, metadata={"asset_rows": 3}) """, "python/checks/orders.py": """ @@ -53,21 +56,25 @@ def orders_asset(ctx): @check(depends_on=export_orders) def check_orders_export(ctx): + '''Test check check_orders_export.''' return ctx.pass_(message="orders exported") @check(depends_on=export_orders, severity="warn") def warn_orders_export(ctx): + '''Test check warn_orders_export.''' return ctx.fail(message="warning check failed") @check(depends_on=orders_asset, tags=("asset",), group="python-checks") def check_orders_asset(ctx): + '''Test check check_orders_asset.''' return ctx.result_of(node_function=orders_asset).payload["asset_rows"] == 3 @check(depends_on=(export_orders, export_customers), tags=("multi",), group="python-checks") def check_order_customer_exports(ctx): + '''Test check check_order_customer_exports.''' return ctx.pass_( metadata={"orders": ctx.result_of(node_function=export_orders).metadata["rows"]} ) @@ -75,16 +82,19 @@ def check_order_customer_exports(ctx): @check(depends_on=[export_orders], severity="error", tags=("failure",), group="python-checks") def fail_orders_export(ctx): + '''Test check fail_orders_export.''' return ctx.fail(message="orders export failed") @check(depends_on=export_orders, severity="error", tags=("failure",), group="python-checks") def false_orders_export(ctx): + '''Test check false_orders_export.''' return False @check(depends_on=export_orders, severity="error", tags=("failure",), group="python-checks") def exception_orders_export(ctx): + '''Test check exception_orders_export.''' raise RuntimeError("orders exception check failed") """, }, @@ -111,11 +121,13 @@ def prepare_terminal_loader_check_project(*, tmp_path: Path) -> Path: @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 1}] """, "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -129,6 +141,7 @@ def raw_orders(ctx): @check(depends_on=raw_orders) def check_raw_orders_loader(ctx): + '''Test check check_raw_orders_loader.''' return True """, }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 332c5d933d..7e3220b3fc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -60,6 +60,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, 100), (2, 200)) @@ -69,7 +70,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, amount_cents FROM __source("raw_orders") """ @@ -77,7 +78,7 @@ + "\n", "models/orders_enriched.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model orders_enriched.", materialized view); SELECT order_id, amount_cents, amount_cents * 2 AS doubled_cents FROM __ref("fact_orders") @@ -86,7 +87,7 @@ + "\n", "models/missing_snapshot.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model missing_snapshot.", materialized table); SELECT 1 AS id """ @@ -202,7 +203,7 @@ def test_given_clone_command_when_running_then_managed_relations_sync_as_expecte ).strip() + "\n", "models/fact_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id, 100 AS amount_cents\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT 1 AS order_id, 100 AS amount_cents\n" ), }, clone_command=( @@ -304,6 +305,7 @@ def test_given_invalid_origin_connection_when_exact_cloning_then_destination_ses @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [] """ ).strip() @@ -312,6 +314,7 @@ def raw_customers(ctx): """ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -324,7 +327,7 @@ def raw_customers(ctx): + "\n", "models/stg_customers.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_customers.", materialized view); SELECT customer_id, first_name FROM __source("raw_customers") """ @@ -384,6 +387,7 @@ def raw_customers(ctx): @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [] """ ).strip() @@ -392,6 +396,7 @@ def raw_customers(ctx): """ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -404,7 +409,7 @@ def raw_customers(ctx): + "\n", "models/stg_customers.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_customers.", materialized view); SELECT customer_id, first_name FROM __source("raw_customers") """ @@ -545,6 +550,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que """ seeds: - name: bonuses + description: Test seed bonuses. schema: seed_origin columns: - name: bonus @@ -554,7 +560,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table, schema model_origin); + MODEL (description "Test model orders.", materialized table, schema model_origin); SELECT * FROM (VALUES (1, 10), (2, 20)) AS orders(order_id, amount) """ @@ -562,7 +568,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "functions/sql/add_bonus.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function add_bonus.", schema scalar_origin, arguments (amount INTEGER), returns INTEGER @@ -574,7 +580,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "functions/sql/order_rows.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function order_rows.", schema table_function_origin, arguments (minimum_id INTEGER), returns table ( @@ -591,7 +597,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "models/enriched_orders.sql": dedent( """ - MODEL (materialized view, schema view_origin); + MODEL (description "Test model enriched_orders.", materialized view, schema view_origin); SELECT order_id, __udf("add_bonus")(amount) AS amount_with_bonus FROM __table_fn("order_rows")(1) @@ -678,6 +684,7 @@ def test_given_python_udf_when_cloning_then_recreates_for_clone_connection_scope runtime_version="3.11", ) def main(value): + '''Test function main.''' return value + 1 """ ).strip() @@ -759,6 +766,7 @@ def test_given_destination_source_deferral_when_cloning_then_function_reads_defe @loader def raw_bonus(ctx): + '''Test loader raw_bonus.''' return [] """ ).strip() @@ -767,6 +775,7 @@ def raw_bonus(ctx): """ sources: - name: raw_bonus + description: Test source raw_bonus. managed: true write_strategy: table columns: @@ -777,7 +786,7 @@ def raw_bonus(ctx): + "\n", "functions/sql/add_deferred_bonus.sql": dedent( """ - FUNCTION (arguments (amount INTEGER), returns INTEGER); + FUNCTION (description "Test function add_deferred_bonus.", arguments (amount INTEGER), returns INTEGER); amount + (SELECT bonus FROM __source("raw_bonus")) """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json index 37c0499799..97bcc86c90 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json @@ -7,7 +7,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\")\n" } }, { @@ -17,7 +17,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q, q + 1 AS q2 FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS q, q + 1 AS q2 FROM __source(\"raw__orders\")\n" } }, { @@ -27,7 +27,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > '2026-01-01'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > '2026-01-01'\n" } }, { @@ -37,7 +37,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ALL\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ALL\n" } }, { @@ -47,7 +47,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (status) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * EXCLUDE (status) FROM __source(\"raw__orders\")\n" } }, { @@ -57,7 +57,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * 2.5 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * 2.5 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -67,7 +67,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id, p.amount_cents FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id, p.amount_cents FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" } }, { @@ -77,7 +77,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q FROM __source(\"raw__orders\") ORDER BY q\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS q FROM __source(\"raw__orders\") ORDER BY q\n" } }, { @@ -87,7 +87,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.customer_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.customer_id)\n" } }, { @@ -97,7 +97,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" } }, { @@ -107,7 +107,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(ordered_at) AS a FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT AVG(ordered_at) AS a FROM __source(\"raw__orders\")\n" } }, { @@ -117,7 +117,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity\n" } }, { @@ -127,7 +127,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NOT quantity AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NOT quantity AS x FROM __source(\"raw__orders\")\n" } }, { @@ -137,7 +137,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at AS x FROM __source(\"raw__orders\") UNION ALL SELECT email FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at AS x FROM __source(\"raw__orders\") UNION ALL SELECT email FROM __source(\"raw__customers\")\n" } }, { @@ -147,7 +147,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT TRUE FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT TRUE FROM __source(\"raw__customers\")\n" } }, { @@ -157,7 +157,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, id FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, id FROM __source(\"raw__orders\")\n" } }, { @@ -167,7 +167,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b) AS (SELECT id FROM __source(\"raw__orders\")) SELECT a FROM c\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH c(a, b) AS (SELECT id FROM __source(\"raw__orders\")) SELECT a FROM c\n" } }, { @@ -177,7 +177,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity + 1 FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity + 1 FROM __source(\"raw__orders\")\n" } }, { @@ -187,7 +187,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status = 'x') AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status = 'x') AS s FROM __source(\"raw__orders\")\n" } }, { @@ -197,7 +197,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity AND status = 'x'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity AND status = 'x'\n" } }, { @@ -207,7 +207,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id\n" } }, { @@ -217,7 +217,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b, d) AS (SELECT id, status FROM __source(\"raw__orders\")) SELECT a FROM c\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH c(a, b, d) AS (SELECT id, status FROM __source(\"raw__orders\")) SELECT a FROM c\n" } }, { @@ -227,7 +227,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" } }, { @@ -237,14 +237,14 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DISTINCT status FROM __source(\"raw__orders\") ORDER BY quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DISTINCT status FROM __source(\"raw__orders\") ORDER BY quantity\n" } }, { "description": "control: declared scalar UDF return type", "category": "control", "repo_files": { - "models/probe.sql": "MODEL (materialized table, contract enforced, columns (valid (type BOOLEAN)));\nSELECT __udf(\"udf__is_completed_order\")(status) AS valid FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table, contract enforced, columns (valid (type BOOLEAN)));\nSELECT __udf(\"udf__is_completed_order\")(status) AS valid FROM __source(\"raw__orders\")\n" }, "expected_exit_code": 0, "expected_codes": [], @@ -257,7 +257,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.id\n" } } ], @@ -271,7 +271,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __source(\"raw__orders\")\n" } }, { @@ -283,7 +283,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.qty FROM __source(\"raw__orders\") o\n" } }, { @@ -295,7 +295,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __ref(\"stg_orders\")\n" } }, { @@ -307,7 +307,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __ref(\"stg_orders_typed\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.qty FROM __ref(\"stg_orders_typed\") o\n" } }, { @@ -319,7 +319,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT amount_usd FROM __source(\"raw__payments\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT amount_usd FROM __source(\"raw__payments\")\n" } }, { @@ -331,7 +331,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT price_usd FROM __seed(\"waffle_types\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT price_usd FROM __seed(\"waffle_types\")\n" } }, { @@ -343,7 +343,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE qty > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE qty > 1\n" } }, { @@ -355,7 +355,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY stat\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY stat\n" } }, { @@ -367,7 +367,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY qty\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY qty\n" } }, { @@ -379,7 +379,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer = c.id\n" } }, { @@ -391,7 +391,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) AS rn FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) AS rn FROM __source(\"raw__orders\")\n" } }, { @@ -403,7 +403,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) = 1\n" } }, { @@ -415,7 +415,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status HAVING SUM(qty) > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status HAVING SUM(qty) > 1\n" } }, { @@ -427,7 +427,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN qty > 1 THEN 'a' END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN qty > 1 THEN 'a' END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -439,7 +439,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(stat) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(stat) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -451,7 +451,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT id FROM __source(\"raw__orders\")) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT id FROM __source(\"raw__orders\")) SELECT quantity FROM x\n" } }, { @@ -463,7 +463,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT s.quantity FROM (SELECT id FROM __source(\"raw__orders\")) s\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT s.quantity FROM (SELECT id FROM __source(\"raw__orders\")) s\n" } }, { @@ -475,7 +475,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.id FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z.id FROM __source(\"raw__orders\") o\n" } }, { @@ -487,7 +487,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.cust)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.cust)\n" } }, { @@ -499,7 +499,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT cust_id FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT cust_id FROM __source(\"raw__customers\"))\n" } }, { @@ -511,7 +511,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE p.amount > 0\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE p.amount > 0\n" } }, { @@ -523,7 +523,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c USING (cust_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c USING (cust_id)\n" } }, { @@ -535,7 +535,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + "models/marts/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" } }, { @@ -547,7 +547,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(stat) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(stat) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -559,7 +559,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" } }, { @@ -571,7 +571,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" } }, { @@ -583,7 +583,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON id = c.id\n" } }, { @@ -595,7 +595,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE order_id > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE order_id > 1\n" } }, { @@ -607,7 +607,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -619,7 +619,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, quantity, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, quantity, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status\n" } }, { @@ -631,7 +631,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE SUM(quantity) > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE SUM(quantity) > 1\n" } }, { @@ -643,7 +643,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(COUNT(*)) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(COUNT(*)) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -655,7 +655,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ROW_NUMBER() OVER () = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ROW_NUMBER() OVER () = 1\n" } }, { @@ -667,7 +667,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, COUNT(*) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, COUNT(*) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -679,7 +679,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING quantity > 1\n" } }, { @@ -691,7 +691,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY 3\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY 3\n" } }, { @@ -703,7 +703,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status, id) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status, id) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -715,7 +715,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT 1 AS id) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT 1 AS id) SELECT quantity FROM x\n" } }, { @@ -727,7 +727,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE qty > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE qty > 1\n" } }, { @@ -739,7 +739,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle = w.waffle_type_id\n" } }, { @@ -751,7 +751,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT waffle_type_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT waffle_type_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" } }, { @@ -763,7 +763,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT order_id FROM __ref(\"stg_orders_typed\")) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT order_id FROM __ref(\"stg_orders_typed\")) SELECT quantity FROM x\n" } }, { @@ -775,7 +775,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -787,7 +787,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT w.price_usd FROM __source(\"raw__orders\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT w.price_usd FROM __source(\"raw__orders\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" } }, { @@ -799,7 +799,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" } }, { @@ -823,7 +823,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type VARCHAR),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type VARCHAR),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -848,8 +848,8 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n qty (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity AS qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n", - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity FROM __ref(\"stg_orders_typed\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n qty (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity AS qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n", + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -885,7 +885,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY SUM(quantity)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY SUM(quantity)\n" } }, { @@ -897,7 +897,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE qty > 1) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) FILTER (WHERE qty > 1) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -909,7 +909,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON SUM(o.customer_id) = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON SUM(o.customer_id) = c.id\n" } }, { @@ -921,7 +921,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ROW_NUMBER() OVER ()) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(ROW_NUMBER() OVER ()) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -933,7 +933,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ROW_NUMBER() OVER ()\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ROW_NUMBER() OVER ()\n" } }, { @@ -945,7 +945,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity) OVER (ORDER BY ordered) AS l FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LAG(quantity) OVER (ORDER BY ordered) AS l FROM __source(\"raw__orders\")\n" } }, { @@ -955,7 +955,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(status, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(status, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -967,7 +967,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.id FROM (SELECT id FROM __source(\"raw__orders\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT x.id FROM (SELECT id FROM __source(\"raw__orders\"))\n" } }, { @@ -979,7 +979,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = z.customer_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = z.customer_id)\n" } }, { @@ -991,7 +991,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id, l.x FROM __source(\"raw__orders\") o, LATERAL (SELECT o.qty AS x) l\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id, l.x FROM __source(\"raw__orders\") o, LATERAL (SELECT o.qty AS x) l\n" } }, { @@ -1003,7 +1003,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" } }, { @@ -1015,7 +1015,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z FROM (SELECT id FROM __source(\"raw__orders\") UNION ALL BY NAME SELECT id FROM __source(\"raw__customers\")) u\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z FROM (SELECT id FROM __source(\"raw__orders\") UNION ALL BY NAME SELECT id FROM __source(\"raw__customers\")) u\n" } }, { @@ -1027,7 +1027,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status ORDER BY quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status ORDER BY quantity\n" } }, { @@ -1039,7 +1039,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")() AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")() AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1051,7 +1051,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.amount FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.amount FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" } }, { @@ -1063,7 +1063,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")(1, 2) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")(1, 2) t\n" } }, { @@ -1075,7 +1075,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(p.amount_usd)') AS d FROM __ref(\"stg_payments\") p\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(p.amount_usd)') AS d FROM __ref(\"stg_payments\") p\n" } }, { @@ -1267,7 +1267,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -1279,7 +1279,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type STRINGY),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type STRINGY),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -1394,7 +1394,7 @@ "description": "metadata-udf: scalar UDF declared argument type", "category": "udf", "repo_files": { - "models/probe.sql": "MODEL (materialized table);\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS valid FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS valid FROM __source(\"raw__orders\")\n" }, "expected_exit_code": 1, "expected_codes": [ @@ -1411,7 +1411,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.email\n" } }, { @@ -1423,7 +1423,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM (SELECT * FROM __source(\"raw__orders\")) AS o JOIN __source(\"raw__customers\") AS c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM (SELECT * FROM __source(\"raw__orders\")) AS o JOIN __source(\"raw__customers\") AS c ON o.id = c.email\n" } } ], @@ -1435,7 +1435,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT * FROM __source(\"raw__orders\")) SELECT qty FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT * FROM __source(\"raw__orders\")) SELECT qty FROM x\n" } }, { @@ -1445,7 +1445,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (qty) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * EXCLUDE (qty) FROM __source(\"raw__orders\")\n" } }, { @@ -1455,7 +1455,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > 5\n" } }, { @@ -1465,7 +1465,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __source(\"raw__orders\")\n" } }, { @@ -1475,7 +1475,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.id = c.email\n" } }, { @@ -1485,7 +1485,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1495,7 +1495,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status > 5\n" } }, { @@ -1505,7 +1505,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status\n" } }, { @@ -1515,7 +1515,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN quantity > 1 THEN quantity ELSE status END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN quantity > 1 THEN quantity ELSE status END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1525,7 +1525,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COALESCE(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1535,7 +1535,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity IN ('a', 'b')\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity IN ('a', 'b')\n" } }, { @@ -1545,7 +1545,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity LIKE '1%'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity LIKE '1%'\n" } }, { @@ -1555,7 +1555,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day', status) AS d FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_TRUNC('day', status) AS d FROM __source(\"raw__orders\")\n" } }, { @@ -1565,7 +1565,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at + status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1575,7 +1575,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LENGTH(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LENGTH(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1585,7 +1585,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at = 'not a date'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at = 'not a date'\n" } }, { @@ -1595,7 +1595,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity - ordered_at AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity - ordered_at AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1605,7 +1605,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at BETWEEN 1 AND 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at BETWEEN 1 AND 5\n" } }, { @@ -1615,7 +1615,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE o.ordered_at > (SELECT MAX(email) FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE o.ordered_at > (SELECT MAX(email) FROM __source(\"raw__customers\"))\n" } }, { @@ -1625,7 +1625,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" } }, { @@ -1635,7 +1635,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(p.payment_method) OVER () AS s FROM __ref(\"stg_payments\") p\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(p.payment_method) OVER () AS s FROM __ref(\"stg_payments\") p\n" } }, { @@ -1645,7 +1645,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1655,7 +1655,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status = TRUE\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status = TRUE\n" } }, { @@ -1665,7 +1665,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS INTEGER) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(ordered_at AS INTEGER) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1675,7 +1675,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE ordered_at > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE ordered_at > 5\n" } }, { @@ -1685,7 +1685,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders\")\n" } }, { @@ -1695,7 +1695,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\")\n" } }, { @@ -1705,7 +1705,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") INTERSECT SELECT id, email FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") INTERSECT SELECT id, email FROM __source(\"raw__customers\")\n" } }, { @@ -1715,7 +1715,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_a_function(id) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT not_a_function(id) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1725,7 +1725,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_TRUNC('day') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1735,7 +1735,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(status, 2) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROUND(status, 2) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1745,7 +1745,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUBSTRING(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUBSTRING(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1755,7 +1755,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY 5\n" } }, { @@ -1765,7 +1765,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" } }, { @@ -1775,7 +1775,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM raw_orders\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM raw_orders\n" } }, { @@ -1785,7 +1785,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST('2020-01-01' AS TIMESTAMP) > 5 AS x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST('2020-01-01' AS TIMESTAMP) > 5 AS x\n" } }, { @@ -1795,7 +1795,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -1815,7 +1815,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ordered_at) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(ordered_at) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1825,7 +1825,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT AVG(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1835,7 +1835,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(CAST(ordered_at AS DATE)) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(CAST(ordered_at AS DATE)) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1845,7 +1845,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity, id) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity, id) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1855,7 +1855,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT STDDEV(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT STDDEV(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1865,7 +1865,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status) AS s FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -1875,7 +1875,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(waffle_name) AS s FROM __seed(\"waffle_types\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(waffle_name) AS s FROM __seed(\"waffle_types\")\n" } }, { @@ -1885,7 +1885,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE status) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) FILTER (WHERE status) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -1895,7 +1895,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROW_NUMBER() AS rn FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROW_NUMBER() AS rn FROM __source(\"raw__orders\")\n" } }, { @@ -1905,7 +1905,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity, status) OVER (ORDER BY id) AS l FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LAG(quantity, status) OVER (ORDER BY id) AS l FROM __source(\"raw__orders\")\n" } }, { @@ -1915,7 +1915,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NTILE('a') OVER (ORDER BY id) AS t FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NTILE('a') OVER (ORDER BY id) AS t FROM __source(\"raw__orders\")\n" } }, { @@ -1925,7 +1925,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER (ORDER BY status RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity) OVER (ORDER BY status RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1935,7 +1935,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER w AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity) OVER w AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1945,7 +1945,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY quantity > 1\n" } }, { @@ -1955,7 +1955,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ABS(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ABS(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1965,7 +1965,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROUND(ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1975,7 +1975,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT EXTRACT(year FROM status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT EXTRACT(year FROM status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1985,7 +1985,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_DIFF('day', status, ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_DIFF('day', status, ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1995,7 +1995,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SPLIT_PART(quantity, ',', 1) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SPLIT_PART(quantity, ',', 1) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2005,7 +2005,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT REGEXP_MATCHES(quantity, 'a') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT REGEXP_MATCHES(quantity, 'a') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2015,7 +2015,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SQRT(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SQRT(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2025,7 +2025,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT GREATEST(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT GREATEST(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2035,7 +2035,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEAST(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LEAST(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2045,7 +2045,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NULLIF(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NULLIF(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2055,7 +2055,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT IFNULL(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT IFNULL(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2065,7 +2065,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT STRFTIME(quantity, '%Y') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT STRFTIME(quantity, '%Y') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2075,7 +2075,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LOWER(ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LOWER(ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2085,7 +2085,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEFT(quantity, 2) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LEFT(quantity, 2) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2095,7 +2095,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE() AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COALESCE() AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2105,7 +2105,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE is_valid(status)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE is_valid(status)\n" } }, { @@ -2115,7 +2115,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_an_agg(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT not_an_agg(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2125,7 +2125,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status OR quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status OR quantity > 1\n" } }, { @@ -2135,7 +2135,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at * 2 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at * 2 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2145,7 +2145,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status % 2 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status % 2 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2155,7 +2155,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status IS TRUE\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status IS TRUE\n" } }, { @@ -2165,7 +2165,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at LIKE '2026%'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at LIKE '2026%'\n" } }, { @@ -2175,7 +2175,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity BETWEEN 'a' AND 'b'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity BETWEEN 'a' AND 'b'\n" } }, { @@ -2185,7 +2185,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN status THEN 1 END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN status THEN 1 END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2195,7 +2195,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT -status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT -status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2205,7 +2205,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at - status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at - status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2215,7 +2215,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING status\n" } }, { @@ -2225,7 +2225,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(quantity AS NOT_A_TYPE) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(quantity AS NOT_A_TYPE) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2235,7 +2235,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(status = 'x' AS TIMESTAMP) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(status = 'x' AS TIMESTAMP) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2245,7 +2245,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS BOOLEAN) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(ordered_at AS BOOLEAN) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2255,7 +2255,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > DATE '2026-13-45'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > DATE '2026-13-45'\n" } }, { @@ -2265,7 +2265,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + INTERVAL 'banana' AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at + INTERVAL 'banana' AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2275,7 +2275,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT id, email FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT id, email FROM __source(\"raw__customers\"))\n" } }, { @@ -2285,7 +2285,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT (SELECT id, email FROM __source(\"raw__customers\") LIMIT 1) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT (SELECT id, email FROM __source(\"raw__customers\") LIMIT 1) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2295,7 +2295,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at IN (SELECT email FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at IN (SELECT email FROM __source(\"raw__customers\"))\n" } }, { @@ -2305,7 +2305,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE (id, customer_id) = (1, 2, 3)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE (id, customer_id) = (1, 2, 3)\n" } }, { @@ -2315,7 +2315,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") o ON o.id = o.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") o ON o.id = o.id\n" } }, { @@ -2325,7 +2325,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM (SELECT id FROM __source(\"raw__orders\")) a JOIN (SELECT email AS id FROM __source(\"raw__customers\")) b USING (id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * FROM (SELECT id FROM __source(\"raw__orders\")) a JOIN (SELECT email AS id FROM __source(\"raw__customers\")) b USING (id)\n" } }, { @@ -2335,7 +2335,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" } }, { @@ -2345,7 +2345,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" } }, { @@ -2355,7 +2355,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")), a AS (SELECT id FROM __source(\"raw__customers\")) SELECT id FROM a\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")), a AS (SELECT id FROM __source(\"raw__customers\")) SELECT id FROM a\n" } }, { @@ -2365,7 +2365,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) SELECT n FROM r\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) SELECT n FROM r\n" } }, { @@ -2375,7 +2375,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM b), b AS (SELECT id FROM __source(\"raw__orders\")) SELECT id FROM a\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM b), b AS (SELECT id FROM __source(\"raw__orders\")) SELECT id FROM a\n" } }, { @@ -2385,7 +2385,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\") ORDER BY status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\") ORDER BY status\n" } }, { @@ -2395,7 +2395,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") EXCEPT SELECT id FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") EXCEPT SELECT id FROM __source(\"raw__customers\")\n" } }, { @@ -2405,7 +2405,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT created_at FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT created_at FROM __source(\"raw__customers\")\n" } }, { @@ -2415,7 +2415,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.* FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z.* FROM __source(\"raw__orders\") o\n" } }, { @@ -2425,7 +2425,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * REPLACE (1 AS qty) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * REPLACE (1 AS qty) FROM __source(\"raw__orders\")\n" } }, { @@ -2435,7 +2435,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COLUMNS('^nothing_') FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COLUMNS('^nothing_') FROM __source(\"raw__orders\")\n" } }, { @@ -2445,7 +2445,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.qty FROM (SELECT * FROM __ref(\"stg_orders\")) x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT x.qty FROM (SELECT * FROM __ref(\"stg_orders\")) x\n" } }, { @@ -2455,7 +2455,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT 'x'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT 'x'\n" } }, { @@ -2465,7 +2465,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT -1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT -1\n" } }, { @@ -2475,7 +2475,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT quantity\n" } }, { @@ -2485,7 +2485,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status) + 1 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status) + 1 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2495,7 +2495,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")('abc') t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")('abc') t\n" } }, { @@ -2505,7 +2505,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(t.waffle_name) AS s FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(t.waffle_name) AS s FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" } }, { @@ -2515,7 +2515,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT UPPER(@cents_to_dollars('p.amount_cents')) AS d FROM __ref(\"stg_payments\") p\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT UPPER(@cents_to_dollars('p.amount_cents')) AS d FROM __ref(\"stg_payments\") p\n" } }, { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql index 3b2df3811b..338a02835f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model typed_orders.", materialized view, contract enforced, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 6c0a84d9c4..d3e3b8b57c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1162,7 +1162,7 @@ def _with_chain_generated_logic(*, sql: str, target_bytes: int) -> str: def _base_model_sql() -> str: return "\n".join( ( - "MODEL (materialized view);", + "MODEL (description 'Test model.', materialized view);", "", "SELECT", " 0 AS id,", @@ -1178,7 +1178,7 @@ def _base_model_sql() -> str: def _chain_model_sql(*, index: int) -> str: previous_model: str = f"model_{index - 1:05d}" - return f'''MODEL (materialized view); + return f'''MODEL (description "Test model.", materialized view); WITH base AS ( SELECT @@ -1242,7 +1242,7 @@ def _chain_model_sql(*, index: int) -> str: def _test_heavy_base_model_sql(*, index: int) -> str: - return f"""MODEL (materialized view); + return f"""MODEL (description "Test model.", materialized view); SELECT {index} AS id, @@ -1253,7 +1253,7 @@ def _test_heavy_base_model_sql(*, index: int) -> str: def _test_heavy_chain_model_sql(*, index: int) -> str: previous_model: str = f"model_{index - 1:05d}" - return f'''MODEL (materialized view); + return f'''MODEL (description "Test model.", materialized view); WITH transformed AS ( SELECT @@ -1459,6 +1459,7 @@ def _layered_write_sources(*, project_dir: Path, source_count: int) -> None: sources_dir.mkdir() entries: str = "\n".join( f""" - name: source_{index:05d} + description: Test source source_{index:05d}. expression: "(SELECT {index} AS id, CAST({index} AS DOUBLE) AS amount, 'source' AS status)" columns: - name: id @@ -1477,6 +1478,7 @@ def _layered_write_seeds(*, project_dir: Path, seed_count: int) -> None: seeds_dir.mkdir() schema_entries: str = "\n".join( f""" - name: seed_{index:05d} + description: Test seed seed_{index:05d}. columns: - name: id type: INTEGER @@ -1497,7 +1499,7 @@ def _layered_write_functions(*, project_dir: Path, function_count: int) -> None: functions_dir.mkdir(parents=True) for index in range(function_count): (functions_dir / f"fn_{index:05d}.sql").write_text( - """FUNCTION ( + """FUNCTION (description "Test function.", arguments (input_value DOUBLE), returns DOUBLE, ); @@ -1566,7 +1568,7 @@ def _layered_write_schemas(*, project_dir: Path) -> None: ) schemas_dir.mkdir(parents=True) (schemas_dir / "benchmark_row.sql").write_text( - """SCHEMA ( + """SCHEMA (description "Test schema.", name benchmark_row, columns ( id (type INTEGER, nullable false), @@ -1626,14 +1628,14 @@ def _layered_model_header(*, index: int, audit_count: int) -> str: id (audits [not_null]), ), );""" - audit_header: str = """MODEL ( + audit_header: str = """MODEL (description "Test model.", columns ( id (nullable false, audits [not_null]), ), );""" return { (True, False): contract_header, - (False, True): "MODEL ();", + (False, True): "MODEL (description 'Test model.');", (False, False): audit_header, }[(index == 0, index >= audit_count)] @@ -1838,7 +1840,11 @@ def _semantic_model_header( audit_sql: str = {False: "", True: ", audits [not_null]"}[column_index < model_audit_count] nullable_sql: str = {False: "", True: ", nullable false"}[column_index == 0] declarations.append(f" {name} (type {data_type}{nullable_sql}{audit_sql})") - return "MODEL (\n columns (\n" + ",\n".join(declarations) + "\n ),\n);" + return ( + 'MODEL (\n description "Generated benchmark model.",\n columns (\n' + + ",\n".join(declarations) + + "\n ),\n);" + ) def _semantic_regular_model_sql( @@ -1958,7 +1964,7 @@ def _semantic_set_operation_model_sql( SHARED_DIAMOND_ROLLUP: str = "shared_orders_rollup" _SHARED_DIAMOND_FOLDER: str = "intermediate/shared_orders" _SHARED_DIAMOND_UPSTREAM_MODEL: str = f"model_{_SPINE_DEPTH - 1:05d}" -_SHARED_DIAMOND_HEADER: str = """MODEL ( +_SHARED_DIAMOND_HEADER: str = """MODEL (description "Test model.", columns ( id (type INTEGER, nullable false), amount (type DOUBLE), @@ -2178,16 +2184,19 @@ def build_rule_gated_test_project_files(*, filler_model_count: int) -> dict[str, ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_refunds\n schema: main\n table: raw_refunds\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_refunds\n description: Test source raw_refunds.\n" + " schema: main\n table: raw_refunds\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n' ), "models/bad_star.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model bad_star.", ' + 'materialized view);\n\nSELECT * FROM __source("raw_orders")\n' ), "tests/unit/orders_case.sql": ( 'TEST (name "orders_case");\n' @@ -2199,7 +2208,8 @@ def build_rule_gated_test_project_files(*, filler_model_count: int) -> dict[str, } for index in range(filler_model_count): files[f"models/filler/filler_{index:03d}.sql"] = ( - f"MODEL (materialized view);\n\nSELECT {index} AS filler_id\n" + "MODEL (description 'Test model.', " + f"materialized view);\n\nSELECT {index} AS filler_id\n" ) return files @@ -2224,23 +2234,24 @@ def build_empty_input_test_project_files( f"[rules.rule_options.SQBRTEST203]\nallowed_tests = {allowed_tests_toml}\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" "SELECT order_id, amount * 2 AS doubled_amount\n" 'FROM __source("raw_orders")\nWHERE amount > 0\n' ), "models/large_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model large_orders.', materialized view);\n\n" 'SELECT order_id FROM __source("raw_orders") WHERE amount > 100\n' ), "models/customers.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model customers.', materialized view);\n\n" 'SELECT order_id AS customer_id FROM __source("raw_orders") WHERE amount > 10\n' ), "models/order_summary.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model order_summary.', materialized view);\n\n" "SELECT COUNT(*) AS order_count, COALESCE(SUM(amount), 0) AS total_amount\n" 'FROM __source("raw_orders")\n' ), @@ -2389,17 +2400,18 @@ def write_relation_stub_project( 'schema = "main"\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT CAST(7 AS INTEGER) AS order_id\n" + "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT CAST(7 AS INTEGER) AS order_id\n" ), "models/order_status.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_status.', materialized table);\n\n" f"WITH {stub_cte_name} AS (\n" " SELECT 'pending' AS order_id, 'pending' AS ghost_status\n" ")\n" f'SELECT {selected_column} FROM __ref("orders")\n' ), "models/next_order.sql": ( - "MODEL (\n" + "MODEL (description 'Test model next_order.',\n" " materialized table,\n" " contract enforced,\n" " columns (order_id (type INTEGER), next_order_id (type INTEGER)),\n" @@ -2423,7 +2435,7 @@ def write_relation_stub_project( "CONSTANT (name min_quantity, value 1);\n" ), "models/marts/channel_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model channel_orders.',\n" " materialized table,\n" " columns (\n" " order_id (nullable false, audits [not_null]),\n" @@ -2686,7 +2698,8 @@ def write_set_operation_project( } for model in models: files[f"models/{model.name}.sql"] = ( - f"MODEL (materialized view, schema analytics);\n\n{model.query_sql}\n" + "MODEL (description 'Test model.', materialized view, schema analytics);\n\n" + f"{model.query_sql}\n" ) prepare_inline_project( tmp_path=project_dir.parent, project_name=project_dir.name, repo_files=files @@ -2776,11 +2789,11 @@ def write_union_fixture_test_project( f'database = "{project_dir / "orders.duckdb"}"\n[rules]\nselect = []\n' ), "models/base_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model base_orders.', materialized view);\n\n" "SELECT 1 AS order_id, CAST(1 AS DOUBLE) AS amount, 'open' AS status\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT b.order_id, b.amount, b.status FROM __ref("base_orders") AS b\n' ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index 0328b31d8f..ddf3c7d886 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -53,7 +53,7 @@ edit=lambda root: write_project_file( root, "models/marts/order_quantities.sql", - 'MODEL ();\n\nSELECT order_id, quantity FROM __ref("stg_orders")\n', + 'MODEL (description "Test model order_quantities.");\n\nSELECT order_id, quantity FROM __ref("stg_orders")\n', ), ), CompileCacheInvalidationTestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py index 378ac50db1..b0c58b50db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py @@ -45,7 +45,7 @@ def test_given_deep_trusted_model_when_compiling_then_sql_analysis_completes( 'schema = "dev"\n' ), "models/deep_expression.sql": ( - "MODEL (columns (resolved_value (type NUMBER)));\n\n" + "MODEL (description 'Test model deep_expression.', columns (resolved_value (type NUMBER)));\n\n" f"SELECT {nested_coalesce_expression(function_depth=test_case.function_depth)} " "AS resolved_value\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py index 8f8df2d770..18978d26f8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py @@ -68,12 +68,12 @@ def test_given_model_selection_when_compiling_then_limits_offline_analysis_and_r "[targets.dev]\n" 'schema = "dev"\n' ), - "models/order_record.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/order_record.sql": "MODEL (description 'Test model order_record.');\n\nSELECT 1 AS order_id\n", "models/result.sql": ( - 'MODEL (columns (order_id ()));\n\nSELECT order_id FROM __ref("order_record")\n' + 'MODEL (description "Test model result.", columns (order_id ()));\n\nSELECT order_id FROM __ref("order_record")\n' ), "models/unrelated.sql": ( - "MODEL (columns (missing_column ()));\n\nSELECT 1 AS present_column\n" + "MODEL (description 'Test model unrelated.', columns (missing_column ()));\n\nSELECT 1 AS present_column\n" ), "selectors.txt": "result\n", }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5dc59b4235..27cb152395 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -42,28 +42,28 @@ 1000, 14.0, 3 * _GIB // 2, - "cfb7bc162018ddc64f9d03ddd2c316e34a333d4129906d8e0b3310399604b980", + "6ea005eef07fe6d2ec47980bc573903efa060cfde55aa5488e63b29ce846280e", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, 45.0, 11 * _GIB // 4, - "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", + "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, 75.0, 13 * _GIB // 4, - "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", + "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, 145.0, 4 * _GIB, - "7a4b6cca4a2fd4b7f5961760177f8b18133854014eaf15a980e8283df646a2cf", + "913a08c2277304efacf75e293728767a3fa416f6b32f5488dca6ab5baea80668", ), ), ids=lambda case: case.description, @@ -141,8 +141,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ 22.0, 24.0, 11 * _GIB // 4, - "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", - "255c19bb188491b53d44c2e5d41472137f763806c699f5acb3899c6b47c5360b", + "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", + "f7f8a3d881496664ec7e293144d93b8b8e486a04eba830d9cc6500d1b28147f3", 3, ), DenseWarmEditCompileGuardTestCase( @@ -152,8 +152,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ 36.0, 38.0, 13 * _GIB // 4, - "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", - "c239f53fb5fa787d250f5f7bf404b3731b480b77d2690dd31fb41ba727475fc0", + "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", + "571fe58493234057c7128b4929d3518532731192f6fef85da0496788826bb172", 3, ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index aa0f9a0f67..4773daca5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -57,16 +57,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( - "f96a10c8d9d218c347432118e069f378140bb6f3a852676d968ad9d5f67d2589" + "7a1978e9f8871b2252abea5e8763c570305f97342b3b3f51e2a2992238cc3798" ), expected_leaf_edit_fingerprint=( - "1e3adee6c9d336de987ea15e3c054726cdf0a00ba8a2b4dcc2f0953b6711cf2c" + "a041d989ef3810e0840361a4f793548b428f941da2f265462a4b46a5bf4d7676" ), expected_macro_edit_fingerprint=( - "81eb4ccff1c67d77a1a4290a239007b92f21bd3112729f64a508592166ed3c39" + "8b62365b60a9f60afaffe6571d6849a187480d503a3114f8783050ebed46363b" ), expected_project_config_fingerprint=( - "c8fbec90f4c03c1bc8c13478121a2ae4c8b9d7c18b0270102130b689a2e76549" + "d077b4676e827b41383cb199f342867995ec1a5564e859e85558a24d1201abf6" ), macro_call_interval=6, scoped_macros=True, @@ -91,16 +91,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( - "26de3005daf1b44463b25165238e83c9a8085b00d942eb4beb1b735ce5f09b57" + "d293f1e9dbe2787d21fa6e5f20c79206ad9beea1e3be63299ac0a5a1e3464ffd" ), expected_leaf_edit_fingerprint=( - "e032f1c0edf2e767f356a6af4891c4dcde0d214d6f691ea1dd0336968e97b707" + "a2e0a328277276f960bef73b4fbc85f9dcacbfad73c10af2df129b78592606ba" ), expected_macro_edit_fingerprint=( - "9e20327a775d2f876f25621d60f8bb81b62d529823a61b8d9686f86de322790c" + "38779c13ac071bf082455f27cdc1d071060da243b9a3d47f0f34b384962b5d11" ), expected_project_config_fingerprint=( - "b65db3cfba6cccef479df505832f7e5821f03b7a0b31e8de967b14796644c768" + "a26f85f97e7c28d40056f95d6be646399b03b215f0a24884f4c26b7b3e996595" ), macro_call_interval=6, scoped_macros=True, @@ -125,16 +125,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( - "b4e5e14d9c1a7ac2e5fc54eac8d1ff2874f2e3042952cb4b80c68c4d4bd1503c" + "8994dd1e87cb97312d492ea66731e4f01f4da08dc2680dd6814f1e4bba23a775" ), expected_leaf_edit_fingerprint=( - "fa0b24b685c7f42a14588ab09e7c929332295ecc83cc9adfdc589b4cbc1c6128" + "f1f25d949e182901656fcef88c7c566254c847174b888bd35c87372aea347807" ), expected_macro_edit_fingerprint=( - "39079f601b624ba4d13d889e9a228c9713ec7ed09ed2127afd3b2c2b48465fbb" + "b8f3cc27552a0d49e97b112732a3b6331040f0dcdec6ed15cc7058a21e98aa4f" ), expected_project_config_fingerprint=( - "6dd7ae799420dcb8a49190356a0b97253b59a3106502cc8cf242ee75c2aedaa2" + "ff3467ff64bfbfa1073a169d53576bfee8699c2529a37250c44e6cc2ecd11845" ), macro_call_interval=6, scoped_macros=True, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index f9e223e1c3..eac764b865 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -39,7 +39,7 @@ expected_max_wall_seconds=17.0, expected_max_rss_bytes=int(1.5 * _GIB), expected_semantic_fingerprint=( - "b763ab23427ed273feb5f80b0b03cf084c7af2f1216500bc5ad7db0be23c2ac1" + "24d42df4968ff361ac0e07e2dbe03abd13bc6948b6812b8188f10d7b0aa9fef4" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -54,7 +54,7 @@ expected_max_wall_seconds=25.0, expected_max_rss_bytes=int(1.75 * _GIB), expected_semantic_fingerprint=( - "fc03f9e22fcf029217e77d3c6e18c9d9945e98e9425a5420a9c4756219a8fd77" + "4fd79eff9a628608a2f9836e1325cec58f9f3d5ac870e284c734f09c505c700f" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -69,7 +69,7 @@ expected_max_wall_seconds=50.0, expected_max_rss_bytes=2 * _GIB, expected_semantic_fingerprint=( - "34e21096cdc9c3a34655f67660de9775b0ac6cebfb5275ce2d647f245ce23f79" + "d83928b7553bcc58564eabc4669045a63e5b314726672ada0a3d979e54ab8573" ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py index c3a809b4ae..cfcd54bb24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py @@ -21,11 +21,11 @@ repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "functions/sql/normalize_status.sql": ( - "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + "FUNCTION (description 'Test function normalize_status.',\n arguments (status VARCHAR),\n returns VARCHAR,\n" " replay_on_change full,\n);\n\nlower(status)\n" ), "models/order_statuses.sql": ( - "MODEL (materialized table);\n\nSELECT __udf(\"normalize_status\")('OPEN') " + "MODEL (description 'Test model order_statuses.', materialized table);\n\nSELECT __udf(\"normalize_status\")('OPEN') " "AS status\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py index 38351db4a1..de78810c9e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py @@ -15,7 +15,7 @@ from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb _STARTS_WITH_FUNCTION: str = ( - "FUNCTION (\n" + "FUNCTION (description 'Test function.',\n" " arguments (value STRING, prefix STRING),\n" " returns BOOLEAN,\n" " database warehouse,\n" @@ -125,7 +125,7 @@ def test_given_builtin_spelling_when_compiling_then_dialect_support_is_enforced( "sqlbuild_project.toml": test_case.project_toml, "functions/sql/starts_with.sql": _STARTS_WITH_FUNCTION, "models/products.sql": ( - f"MODEL (database warehouse, schema analytics);\n\n{test_case.query_sql}" + f"MODEL (description 'Test model products.', database warehouse, schema analytics);\n\n{test_case.query_sql}" ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py index 35980a6788..ecfeab998f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py @@ -34,6 +34,7 @@ """ seeds: - name: countries + description: Test seed countries. columns: - name: code type: VARCHAR @@ -52,11 +53,11 @@ "from sqlbuild.functions import udf\n\n" '@udf(arguments={}, returns="INTEGER", runtime_version="3.11")\n' "def main():\n" - " return 42\n" + " '''Test function main.'''\n return 42\n" ) _MANAGED_SOURCE: str = ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -67,7 +68,7 @@ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ) @@ -78,7 +79,7 @@ description="model without logical schema fails under preserve", repo_files={ "sqlbuild_project.toml": _PRESERVE_CONFIG, - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Model 'orders' has no logical schema", @@ -97,7 +98,7 @@ description="SQL scalar function without logical schema fails under preserve", repo_files={ "sqlbuild_project.toml": _PRESERVE_CONFIG, - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", }, expected_exit_code=1, expected_stderr_fragment="SQL function 'answer' has no logical schema", @@ -108,7 +109,7 @@ "sqlbuild_project.toml": _PRESERVE_CONFIG, "functions/sql/answers.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function answers.", returns table ( answer INTEGER ) @@ -132,6 +133,7 @@ @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -174,16 +176,17 @@ def main(): """ ).strip() + "\n", - "models/marts/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "functions/python/python_answer.py": dedent( """ from sqlbuild.functions import udf @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -198,16 +201,17 @@ def main(): "sqlbuild_project.toml": _PRESERVE_CONFIG.replace( 'schema = "preserve"', 'schema = "physical"' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "functions/python/python_answer.py": dedent( """ from sqlbuild.functions import udf @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -220,7 +224,7 @@ def main(): description="warehouse model without explicit schema fails offline", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Model 'orders' has no explicitly resolved physical write schema", @@ -239,7 +243,7 @@ def main(): description="warehouse SQL scalar function without explicit schema fails offline", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", }, expected_exit_code=1, expected_stderr_fragment="Function 'answer' has no explicitly resolved physical write schema", @@ -249,7 +253,7 @@ def main(): repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, "functions/sql/answers.sql": ( - "FUNCTION (returns table (answer INTEGER));\n\nSELECT 42 AS answer\n" + "FUNCTION (description 'Test function answers.', returns table (answer INTEGER));\n\nSELECT 42 AS answer\n" ), }, expected_exit_code=1, @@ -280,7 +284,7 @@ def main(): description="resource schema supplies physical namespace", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -291,7 +295,7 @@ def main(): "sqlbuild_project.toml": ( _POSTGRES_CONFIG_WITHOUT_SCHEMA + '\n[defaults]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -305,7 +309,7 @@ def main(): ) + '\n[targets.dev]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -316,7 +320,7 @@ def main(): "sqlbuild_project.toml": ( _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA + 'schema = "preserve"\n' ), - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -325,7 +329,7 @@ def main(): description="named target rejects model logical schema as strategy", repo_files={ "sqlbuild_project.toml": _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment=( @@ -338,7 +342,7 @@ def main(): "sqlbuild_project.toml": ( _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA + '\n[defaults]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Named target 'dev' must explicitly set schema", @@ -350,7 +354,7 @@ def main(): 'database = "analytics"', 'database = "analytics"\nschema = "connection_schema"', ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Named target 'dev' must explicitly set schema", @@ -361,7 +365,7 @@ def main(): "sqlbuild_project.toml": ( _POSTGRES_CONFIG_WITHOUT_SCHEMA + 'schema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -389,10 +393,10 @@ def main(): "[connection]\n" 'database = ":memory:"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "sources/raw.yml": _MANAGED_SOURCE, "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, @@ -408,7 +412,7 @@ def main(): 'database = ":memory:"\n\n' "[targets.dev]\n" ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py index 31027c2ed9..07e4f9ad95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py @@ -37,7 +37,7 @@ """ ).strip() + "\n", - "models/commerce/partner_feed/staging/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/commerce/partner_feed/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -61,7 +61,7 @@ """ ).strip() + "\n", - "models/commerce/eu/staging/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/commerce/eu/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment=( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index b01925c069..1df9bffb70 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -169,7 +169,7 @@ def test_given_python_under_unsupported_root_when_compiling_then_command_fails_w description="custom Rule harness test path", repo_files={ "sqlbuild_project.toml": 'name = "rule_harness_layout"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", "tests/rules/test_order_policy.py": "def test_order_policy(): pass\n", }, expected_exit_code=0, @@ -215,7 +215,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -225,7 +225,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte " type: VARCHAR\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, status FROM __source("raw_orders")\n' ), "python/_helpers.py": ( @@ -245,14 +245,14 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return order_rows()\n" + " '''Test loader raw_orders.'''\n return order_rows()\n" ), "python/orders/exports/assets.py": ( "from sqlbuild.assets import asset\n" "from sqlbuild.refs import model\n\n\n" "@asset(depends_on=model('fact_orders'))\n" "def orders_export(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" + " '''Test asset orders_export.'''\n relation = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()\n" " return ctx.result(payload={'order_count': int(rows[0][0])})\n" ), @@ -261,7 +261,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte "from sqlbuild.checks import check\n\n\n" "@check(depends_on=orders_export)\n" "def orders_export_not_empty(ctx):\n" - " payload = ctx.result_of(node_function=orders_export).payload\n" + " '''Test check orders_export_not_empty.'''\n payload = ctx.result_of(node_function=orders_export).payload\n" " if payload['order_count'] <= 0:\n" " return ctx.fail(message='orders export is empty')\n" " return ctx.pass_(message='orders export is ready')\n" @@ -322,7 +322,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return None\n\n\n" + " '''Test task orders.'''\n return None\n\n\n" "@factory\n" "def order_nodes():\n" " return orders\n" @@ -342,7 +342,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi " global orders\n\n" " @task\n" " def orders(ctx):\n" - " return None\n\n" + " '''Test task orders.'''\n return None\n\n" " return orders\n" ), }, @@ -386,12 +386,12 @@ def test_given_factory_returning_module_node_when_planning_then_node_registers_o "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return None\n\n\n" + " '''Test task orders.'''\n return None\n\n\n" "@factory\n" "def order_nodes():\n" " @task(name='orders')\n" " def generated_orders(ctx):\n" - " return None\n" + " '''Test task generated_orders.'''\n return None\n" " return generated_orders\n" ), }, @@ -441,7 +441,7 @@ def test_given_factory_returning_distinct_same_name_node_when_planning_then_conf "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return ctx.result(payload={'status': python.helpers.clean.label()})\n" + " '''Test task orders.'''\n return ctx.result(payload={'status': python.helpers.clean.label()})\n" ), }, expected_build_fragments=("orders", "\u2713 Completed successfully"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py index a81c41c446..be9fdc3e92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py @@ -37,8 +37,8 @@ def test_given_open_source_when_planning_then_live_columns_reject_missing_refere project_name="source_binding", repo_files={ "sqlbuild_project.toml": 'name = "source_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "sources/orders.yml": "sources:\n - name: raw_orders\n table: raw_orders\n", - "models/orders.sql": 'MODEL (materialized table);\nSELECT missing FROM __source("raw_orders")\n', + "sources/orders.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: raw_orders\n", + "models/orders.sql": 'MODEL (description "Test model orders.", materialized table);\nSELECT missing FROM __source("raw_orders")\n', }, ) database: Path = project / "orders.duckdb" @@ -77,7 +77,7 @@ def test_given_invalid_metadata_when_planning_then_no_warehouse_connection_is_op project_name="metadata_binding", repo_files={ "sqlbuild_project.toml": 'name = "metadata_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "models/orders.sql": "MODEL (materialized table, unique_key [missing]);\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table, unique_key [missing]);\nSELECT 1 AS id\n", }, ) result: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py index 3c267d704f..688cd7f480 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py @@ -22,9 +22,9 @@ description="misspelled MODEL key", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", "models/fct_orders.sql": ( - 'MODEL (\n materialized table,\n descripton "Orders fact.",\n);\n\n' + 'MODEL (description "Test model fct_orders.",\n materialized table,\n descripton "Orders fact.",\n);\n\n' 'SELECT order_id FROM __ref("stg_orders")\n' ), }, @@ -40,7 +40,7 @@ "sqlbuild_project.toml": ( _PROJECT_TOML + '\n[path_defaults.staging]\nmaterialised = "view"\n' ), - "models/staging/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/staging/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\nSELECT 1 AS order_id\n", }, expected_fragments=( "error[D001]", @@ -53,11 +53,11 @@ repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n" " columns:\n - name: order_id\n tpye: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' ), }, expected_fragments=( @@ -71,7 +71,7 @@ repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/fct_orders.sql": ( - "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + "MODEL (description 'Test model fct_orders.',\n materialized incremental,\n incremental_strategy append,\n" " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" ), }, @@ -89,7 +89,7 @@ _PROJECT_TOML + '\n[defaults]\nreplay_on_change = "everything"\n' ), "models/fct_orders.sql": ( - "MODEL (materialized incremental, incremental_strategy append);\n\n" + "MODEL (description 'Test model fct_orders.', materialized incremental, incremental_strategy append);\n\n" "SELECT 1 AS order_id\n" ), }, @@ -110,7 +110,7 @@ " - name: raw_orders\n table: orders\n" " primary_keys: order_id\n" ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\nSELECT 1 AS order_id\n", }, expected_fragments=( "error[D006]", @@ -151,7 +151,7 @@ def test_given_unknown_model_config_key_when_compiling_then_fails_naming_key_and " return MaterializationResult(success=True)\n" ), "models/order_snapshot.sql": ( - "MODEL (\n materialized order_snapshot,\n" + "MODEL (description 'Test model order_snapshot.',\n materialized order_snapshot,\n" " config (retention_days 7, partition_column order_id),\n);\n\n" "SELECT 1 AS order_id\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py index ef9a5f99ff..b079f15144 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -44,10 +44,10 @@ 150, 250, ( - "a0fa536e6e64d9be7027ca9a52b4e84431361ec4598aeff1c721fe266fdb096d", - "50e52f18c8bc90479f83d744594cbdefeb13e26a3740ef9bb2ef077f15d2cb5a", - "a08f6c4470d65e2964c10f9c41f6d59ef92089bc5d5bea80104495edb6080d99", - "30809e387b0b096fc592f7da6e429c0c4878c41ed9cc5ea579898fd7e1029687", + "8ef4dcbf6abc8750aee9ae740b09bfd442e6593f5176f059bdc2e1ce01ec538c", + "7c3a736acd69d910a3c612180c982e44a8a07473713f533ab8c5a90a7ecee3e4", + "43a14caf30367ca9e2b966d20ab98b83b2ab8fb36312956f7a11bf001944a0a6", + "b041d70eef587492ab936e98d527fd76895eff69e3d5224e48419e73ed426bf3", ), ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py index 62291c6de3..e5de4a552f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py @@ -271,7 +271,8 @@ def prepare_databricks_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: models_dir: Path = project_dir / "models" / "staging" models_dir.mkdir(parents=True, exist_ok=True) (models_dir / "stg_orders.sql").write_text( - "MODEL (materialized table, unique_key [order_id]);\n\n" + "MODEL (description 'Test model stg_orders.', " + "materialized table, unique_key [order_id]);\n\n" "SELECT * FROM (" "SELECT 1 AS order_id, 1 AS customer_id, 100 AS amount_cents UNION ALL " "SELECT 2 AS order_id, 2 AS customer_id, 200 AS amount_cents" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 9f3b1c19f7..074f14f4a7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -166,14 +166,14 @@ def test_given_python_result_when_running_check_on_databricks_then_persists_node "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -248,13 +248,13 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" database: {catalog_name}\n" f" schema: {schema_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -292,7 +292,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -301,7 +301,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -327,7 +327,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -338,7 +338,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -359,7 +359,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -370,7 +370,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -395,7 +395,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -606,7 +606,7 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ DatabricksScenarioLocalReplayE2ETestCase( description="captures databricks fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " date_trunc('DAY', event_ts) AS event_day,\n" @@ -616,7 +616,7 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ "GROUP BY customer_id, date_trunc('DAY', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TIMESTAMP '2026-01-01 08:15:00' " @@ -647,12 +647,12 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ description="reports databricks local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -675,14 +675,14 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ description="reports databricks local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(amount_cents) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -1391,7 +1391,7 @@ def test_given_databricks_invalid_model_when_building_then_underlying_error_is_p project_dir, schema_name = prepare_databricks_waffle_shop(tmp_path=tmp_path) broken_model: Path = project_dir / "models" / "marts" / "databricks_broken_model.sql" broken_model.write_text( - "MODEL (materialized table);\n\nSELECT missing_column FROM (SELECT 1 AS id)", + "MODEL (description 'Test model databricks_broken_model.', materialized table);\n\nSELECT missing_column FROM (SELECT 1 AS id)", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py index 9a2ec73d7f..b1c5dcfaff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py @@ -66,11 +66,12 @@ def test_given_dlt_source_when_loading_to_databricks_then_table_is_materialized_ f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py index e8ab8658ae..e79a4f0c04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -120,11 +120,13 @@ def write_sqlbuild_defer_target_models(*, project_dir: Path) -> None: encoding="utf-8", ) project_dir.joinpath("models", "deferred_upstream.sql").write_text( - "MODEL (materialized table);\n\nSELECT 42 AS order_id\n", + "MODEL (description 'Test model deferred_upstream.', " + "materialized table);\n\nSELECT 42 AS order_id\n", encoding="utf-8", ) project_dir.joinpath("models", "deferred_consumer.sql").write_text( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("deferred_upstream")\n', + 'MODEL (description "Test model deferred_consumer.", ' + 'materialized table);\n\nSELECT order_id FROM __ref("deferred_upstream")\n', encoding="utf-8", ) @@ -233,7 +235,8 @@ def write_sqlbuild_order_status_snapshot(*, project_dir: Path) -> None: encoding="utf-8", ) project_dir.joinpath("models", "order_status_history.sql").write_text( - "MODEL (\n materialized snapshot,\n unique_key [order_id],\n" + "MODEL (description 'Test model order_status_history.'," + "\n materialized snapshot,\n unique_key [order_id],\n" " snapshot_strategy check,\n check_columns [status],\n);\n\n" "SELECT 10 AS order_id, 'open' AS status\n", encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py index 9f48c6bb72..9999bfdeaa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -129,7 +129,7 @@ def test_given_duckdb_dbt_project_when_running_dbt_init_then_generated_project_b (sqlbuild_project_dir / "models").mkdir() (sqlbuild_project_dir / "models" / "local_profile_orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 2 AS order_id\n", + "MODEL (description 'Test model local_profile_orders.', materialized table);\n\nSELECT 2 AS order_id\n", encoding="utf-8", ) @@ -146,7 +146,7 @@ def test_given_duckdb_dbt_project_when_running_dbt_init_then_generated_project_b ) == [(2,)] (sqlbuild_project_dir / "models" / "downstream_orders.sql").write_text( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model downstream_orders.', materialized table);\n\n" 'SELECT order_id FROM __dbt_ref("analytics", "dbt_orders")\n', encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py index 4a799e9533..7d68725aa6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py @@ -46,7 +46,8 @@ def test_given_target_model_limit_when_running_dbt_interop_then_refuses_sqlbuild target_settings="\n[targets.main.execution_limits]\nmax_models = 1\n", ) project_dir.joinpath("models", "local_order_ids.sql").write_text( - "MODEL (materialized table);\n\nSELECT 11 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model local_order_ids.', materialized table);\n\nSELECT 11 AS order_id\n", + encoding="utf-8", ) result: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index 504e508064..ef2e0c5869 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -287,6 +287,7 @@ def test_given_debug_command_with_providers_when_running_then_outputs_provider_s class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "debug" """ ).strip() @@ -346,6 +347,7 @@ def test_given_debug_json_with_providers_when_running_then_outputs_provider_summ class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "debug" """ ).strip() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py index b973ae1183..9d54225c5c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py @@ -50,6 +50,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, 1, TIMESTAMP '2026-04-01 09:15:00', 'placed', 100), @@ -61,7 +62,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT order_id, @@ -75,7 +76,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/orders_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_snapshot.", materialized table, unique_key [order_id], cursor ordered_at, @@ -102,7 +103,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/customer_totals.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_totals.", materialized table, unique_key [customer_id] ); @@ -118,7 +119,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/orders_sparse.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_sparse.", materialized table, unique_key [order_id] ); @@ -133,7 +134,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/marts/daily_revenue.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model daily_revenue.", materialized table); SELECT CAST(ordered_at AS DATE) AS revenue_date, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py index 876eacafb8..93b6ce36ab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py @@ -30,7 +30,8 @@ def write_format_performance_project( tests_dir.mkdir(parents=True) for model_index in range(model_count): models_dir.joinpath(f"orders_{model_index:05d}.sql").write_text( - "MODEL (columns (order_id (type INTEGER), note (type VARCHAR)));\n" + 'MODEL (description "Test model.", ' + "columns (order_id (type INTEGER), note (type VARCHAR)));\n" f"SELECT {model_index} AS order_id, 'ready' AS note\n", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py index ea215f1c40..08cbb50548 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py @@ -21,7 +21,7 @@ [ RuleFixTestCase( "verification failure", - "MODEL (sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item", + 'MODEL (description "Test model.", sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item', "refused", "SQBRSQL005", 1, @@ -30,7 +30,7 @@ ), RuleFixTestCase( "generated dependency region", - "MODEL ();\n" + 'MODEL (description "Test model.");\n' 'WITH unused AS (SELECT customer_id FROM __ref("customers")) SELECT 2 AS item', "refused", "SQBRSQL005", @@ -40,7 +40,7 @@ ), RuleFixTestCase( "fixed point with overlapping unused CTE deletions", - "MODEL ();\n" + 'MODEL (description "Test model.");\n' "WITH first AS (SELECT 1 AS item), second AS (SELECT item FROM first), " "third AS (SELECT item FROM second) SELECT 2 AS item", "applied", @@ -59,7 +59,9 @@ def test_given_rule_fixes_when_running_cli_then_reports_verified_or_refused_chan model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text(test_case.sql) - (model.parent / "customers.sql").write_text("MODEL (); SELECT 1 AS customer_id") + (model.parent / "customers.sql").write_text( + 'MODEL (description "Test model customers."); SELECT 1 AS customer_id' + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -104,7 +106,9 @@ def test_given_wide_or_deep_unused_ctes_when_fixing_then_work_is_bounded( ctes: list[str] = [f"items_0 AS (SELECT {columns})"] for index in range(1, test_case.depth): ctes.append(f"items_{index} AS (SELECT * FROM items_{index - 1})") - model.write_text("MODEL (); WITH " + ", ".join(ctes) + " SELECT 1 AS order_id") + model.write_text( + 'MODEL (description "Test model."); WITH ' + ", ".join(ctes) + " SELECT 1 AS order_id" + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py index 3283901d5c..66db7920be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py @@ -26,7 +26,8 @@ def prepare_freshness_project( include_managed_source=include_managed_source, ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), } repo_files.update( @@ -37,7 +38,7 @@ def prepare_freshness_project( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_managed(ctx):\n" - " return [{'event_id': 5}]\n" + " '''Test loader raw_managed.'''\n return [{'event_id': 5}]\n" ) }, }[include_managed_source] @@ -62,10 +63,12 @@ def prepare_multi_schema_freshness_project(*, tmp_path: Path) -> Path: ), "sources/raw.yml": freshness_sources_yml(include_error_source=False), "models/dev_orders.sql": ( - 'MODEL (materialized table, schema dev);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model dev_orders.", ' + 'materialized table, schema dev);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/mart_orders.sql": ( - 'MODEL (materialized table, schema mart);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model mart_orders.", ' + 'materialized table, schema mart);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -88,6 +91,7 @@ def freshness_sources_yml( False: "", True: ( " - name: raw_error\n" + " description: Test source raw_error.\n" " expression: SELECT 3 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -99,6 +103,7 @@ def freshness_sources_yml( False: "", True: ( " - name: raw_managed\n" + " description: Test source raw_managed.\n" " managed: true\n" " expression: SELECT 5 AS event_id\n" " freshness:\n" @@ -109,12 +114,13 @@ def freshness_sources_yml( }[include_managed_source] return ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" f"{freshness}" f"{error_source}" f"{managed_source}" " - name: raw_unknown\n" + " description: Test source raw_unknown.\n" " expression: SELECT 2 AS order_id\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py index a5b67872a8..4706f2ae5f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py @@ -70,7 +70,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir description="initialized project builds Python nodes and helpers under python", project_files={ "models/marts/fact_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT 1 AS order_id\n" ), "python/_helpers.py": "def order_count(rows):\n return len(rows)\n", "python/tasks/orders.py": ( @@ -78,7 +78,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir "from sqlbuild.tasks import task\n\n\n" "@task\n" "def count_orders(ctx):\n" - " return ctx.result(payload={'order_count': order_count([1, 2])})\n" + " '''Test task count_orders.'''\n return ctx.result(payload={'order_count': order_count([1, 2])})\n" ), "python/assets/orders_export.py": ( "from python.tasks.orders import count_orders\n" @@ -86,7 +86,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir "from sqlbuild.refs import model\n\n\n" "@asset(depends_on=(model('fact_orders'), count_orders))\n" "def orders_export(ctx):\n" - " return ctx.result(payload={'ready': True})\n" + " '''Test asset orders_export.'''\n return ctx.result(payload={'ready': True})\n" ), }, unexpected_paths=("tasks", "assets", "checks", "loaders", "factories", "libs"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 68823f71ca..e50ef55753 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -59,7 +59,7 @@ def prepare_janitor_project( "sqlbuild_project.toml": project_config, "models/orders.sql": dedent( """ - MODEL (); + MODEL (description "Test model orders."); SELECT 1 AS order_id """ @@ -285,7 +285,8 @@ def prepare_archive_janitor_project( 'materialized = "table"\n' ), **{ - f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + f"models/{model_name}.sql": f"MODEL (description 'Test model.');\n\n" + f"SELECT 1 AS {model_name}_id\n" for model_name in model_names }, f"adapters/{AGED_JANITOR_ADAPTER_NAME}.py": AGED_JANITOR_ADAPTER_SOURCE, @@ -393,7 +394,8 @@ def prepare_two_target_janitor_project( 'materialized = "table"\n' ), **{ - f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + f"models/{model_name}.sql": f"MODEL (description 'Test model.');\n\n" + f"SELECT 1 AS {model_name}_id\n" for model_name in model_names }, }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 1be089a3c1..810db559b5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -202,12 +202,13 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ """ ).strip() + "\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", - "models/safe_orders.sql": "MODEL (schema safe,);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", + "models/safe_orders.sql": "MODEL (description 'Test model safe_orders.', schema safe,);\n\nSELECT 1 AS order_id\n", "sources/raw.yml": dedent( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index 8aa3fa0063..dc0169530d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -128,7 +128,8 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i assert origin_build.returncode == 0, origin_build.stdout + origin_build.stderr (project_dir / "models" / "revenue.sql").unlink() (project_dir / "models" / "daily_revenue.sql").write_text( - "MODEL (\n migrate_from revenue,\n);\n\nSELECT 1 AS revenue_id\n", encoding="utf-8" + "MODEL (description 'Test model daily_revenue.',\n migrate_from revenue,\n);\n\nSELECT 1 AS revenue_id\n", + encoding="utf-8", ) rename_build: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", "--target", "prod"), project_dir=project_dir diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py index 3ec718855d..0603f89712 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py @@ -13,13 +13,16 @@ def prepare_lineage_cache_project(*, tmp_path: Path) -> Path: repo_files={ "sqlbuild_project.toml": 'name = "lineage_cache_project"\nadapter = "duckdb"\n', "models/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS order_id, 10 AS customer_id\n" + "MODEL (description 'Test model stg_orders.', " + "materialized view);\n\nSELECT 1 AS order_id, 10 AS customer_id\n" ), "models/stg_customers.sql": ( - "MODEL (materialized view);\n\nSELECT 10 AS customer_id\n" + "MODEL (description 'Test model stg_customers.', " + "materialized view);\n\nSELECT 10 AS customer_id\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", ' + 'materialized view);\n\nSELECT * FROM __ref("stg_orders")\n' ), }, ) @@ -49,16 +52,18 @@ def prepare_diamond_lineage_project(*, tmp_path: Path) -> Path: files: dict[str, str] = { "sqlbuild_project.toml": 'name = "diamond_lineage"\nadapter = "duckdb"\n', - "models/orders_0.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id, 10 AS amount\n", + "models/orders_0.sql": "MODEL (description 'Test model orders_0.', " + "materialized view);\n\nSELECT 1 AS order_id, 10 AS amount\n", } for layer in range(1, DIAMOND_LAYERS + 1): previous: str = f"orders_{layer - 1}" for side in ("left", "right"): files[f"models/orders_{side}_{layer}.sql"] = ( - f'MODEL (materialized view);\n\nSELECT order_id, amount FROM __ref("{previous}")\n' + 'MODEL (description "Test model.", ' + f'materialized view);\n\nSELECT order_id, amount FROM __ref("{previous}")\n' ) files[f"models/orders_{layer}.sql"] = ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model.', materialized view);\n\n" "SELECT l.order_id, l.amount + r.amount AS amount\n" f'FROM __ref("orders_left_{layer}") AS l\n' f'JOIN __ref("orders_right_{layer}") AS r ON l.order_id = r.order_id\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py index e42776e88d..27d9a44aab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py @@ -105,7 +105,7 @@ def test_given_directional_selectors_when_expanding_then_depth_and_exclusions_ar for name in ("stg_orders.sql", "stg_customers.sql"): (project_dir / "models" / name).rename(staging_dir / name) (project_dir / "models" / "fact_orders.sql").write_text( - 'MODEL (materialized view, tags [finance]);\nSELECT * FROM __ref("stg_orders")\n', + 'MODEL (description "Test model fact_orders.", materialized view, tags [finance]);\nSELECT * FROM __ref("stg_orders")\n', encoding="utf-8", ) @@ -336,7 +336,7 @@ def test_given_authored_dependency_change_when_lineage_runs_then_rebuilds_struct ) assert first.returncode == 0, first.stdout + first.stderr (project_dir / "models/fact_orders.sql").write_text( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model fact_orders.', materialized view);\n\n" 'SELECT orders.order_id FROM __ref("stg_orders") AS orders\n' 'JOIN __ref("stg_customers") AS customers USING (customer_id)\n', encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py index 37704c6616..a35a991aea 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py @@ -30,7 +30,10 @@ def assert_source_only_ingress_dependency_case( project_name="source_loader_schema_behavior", repo_files={ **build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n" "from python.tasks.prepare import prepare_events\n\n" @@ -39,13 +42,13 @@ def assert_source_only_ingress_dependency_case( " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [\n" + " '''Test loader fetch_events.'''\n return [\n" " {'event_id': 1, 'load_seq': 1},\n" " {'event_id': 2, 'load_seq': 1},\n" " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), @@ -55,6 +58,7 @@ def assert_source_only_ingress_dependency_case( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" + " '''Test task prepare_events.'''\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text('prepared')\n" " return ctx.result()\n" ), @@ -233,7 +237,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -249,7 +253,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " type: INTEGER\n" " - name: load_seq\n" " type: INTEGER\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -270,7 +274,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_order_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_order_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -301,7 +305,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_customers(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_customers.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'customer_id': 1, 'plan_name': 'basic', 'load_seq': 1},\n" " {'customer_id': 2, 'plan_name': 'plus', 'load_seq': 1},\n" @@ -316,6 +320,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_prices(ctx):\n" + " '''Test loader fetch_prices.'''\n" " classic_price = 600 if ctx.current_cursor_value is None else 650\n" " return [\n" " {'waffle_type': 'classic', 'price_cents': classic_price, 'load_seq': 1},\n" @@ -323,7 +328,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n\n" "@loader(depends_on=[fetch_order_events, fetch_prices])\n" "def raw_orders(ctx):\n" - " events = ctx.loader(fetch_order_events)\n" + " '''Test loader raw_orders.'''\n events = ctx.loader(fetch_order_events)\n" " prices = ctx.loader(fetch_prices)\n" " cursor = ctx.query(\n" " f'SELECT e.order_id, e.customer_id, e.waffle_type, e.quantity, '\n" @@ -344,7 +349,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n\n" "@loader(depends_on=[fetch_customers])\n" "def raw_customers(ctx):\n" - " customers = ctx.loader(fetch_customers)\n" + " '''Test loader raw_customers.'''\n customers = ctx.loader(fetch_customers)\n" " cursor = ctx.query(\n" " f'SELECT customer_id, plan_name, load_seq FROM {customers.destination} '\n" " f'ORDER BY customer_id'\n" @@ -355,7 +360,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n" ), "models/fact_waffle_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_waffle_orders.', materialized table);\n\n" "SELECT\n" " o.order_id,\n" " o.customer_id,\n" @@ -369,7 +374,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - 'LEFT JOIN __source("raw_customers") c ON o.customer_id = c.customer_id\n' ), "models/customer_revenue.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_revenue.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " COALESCE(plan_name, 'unknown') AS plan_name,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py index dca6faccec..21adbee791 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py @@ -58,11 +58,12 @@ def test_given_dlt_sql_database_source_when_loading_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, @@ -132,6 +133,7 @@ def test_given_dlt_filesystem_source_when_loading_then_table_is_materialized_and ' file_glob: "orders.jsonl"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " reader: jsonl\n" " write_disposition: replace\n" ), @@ -141,7 +143,7 @@ def test_given_dlt_filesystem_source_when_loading_then_table_is_materialized_and '{"order_id": 3, "amount": 30}\n' ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, @@ -210,13 +212,14 @@ def test_given_dlt_rest_api_source_when_loading_then_table_is_materialized_and_q f' base_url: "{base_url}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " endpoint:\n" ' path: "orders"\n' ' data_selector: "$"\n' " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 255c678e4a..40ffb4dc45 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -133,7 +133,7 @@ def test_given_non_contract_loader_when_late_column_appears_then_existing_target repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -147,7 +147,7 @@ def test_given_non_contract_loader_when_late_column_appears_then_existing_target "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -309,10 +309,10 @@ def test_given_two_terminal_sources_when_sharing_intermediate_then_intermediate_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" - " - name: raw_order_metrics\n" + " - name: raw_order_metrics\n description: Test source raw_order_metrics.\n" " managed: true\n" " write_strategy: table\n" ), @@ -323,21 +323,21 @@ def test_given_two_terminal_sources_when_sharing_intermediate_then_intermediate_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_orders(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_orders.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" " return [{'event_id': next_seq, 'load_seq': next_seq}]\n\n" "@loader(depends_on=[fetch_orders])\n" "def raw_orders(ctx):\n" - " orders = ctx.loader(fetch_orders)\n" + " '''Test loader raw_orders.'''\n orders = ctx.loader(fetch_orders)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {orders.destination} ORDER BY event_id'\n" " )\n" " return [{'event_id': row[0]} for row in cursor.fetchall()]\n\n" "@loader(depends_on=[fetch_orders])\n" "def raw_order_metrics(ctx):\n" - " orders = ctx.loader(fetch_orders)\n" + " '''Test loader raw_order_metrics.'''\n orders = ctx.loader(fetch_orders)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {orders.destination} ORDER BY event_id'\n" " )\n" @@ -392,7 +392,7 @@ def test_given_chained_loader_project_when_loading_source_then_runs_dependencies repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -407,14 +407,14 @@ def test_given_chained_loader_project_when_loading_source_then_runs_dependencies " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " source_cursor = ctx.source('raw_events').max('event_id')\n" + " '''Test loader fetch_events.'''\n source_cursor = ctx.source('raw_events').max('event_id')\n" " return [\n" " {'event_id': 1, 'load_seq': 1},\n" " {'event_id': 2, 'load_seq': 1},\n" " ] if source_cursor is None else []\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status, {events.current_cursor_value} AS max_seq " 'FROM {events.destination}")\n' @@ -464,7 +464,7 @@ def test_given_chained_loader_project_when_building_source_model_then_runs_loade repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -478,17 +478,17 @@ def test_given_chained_loader_project_when_building_source_model_then_runs_loade " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status FROM {events.destination}\")\n" ), ) | { "models/fact_events.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_events.', materialized table);\n\n" 'SELECT event_id, status FROM __source("raw_events")\n' ) }, @@ -543,7 +543,7 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -557,17 +557,17 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status FROM {events.destination}\")\n" ), ) | { "models/fact_events.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_events.', materialized table);\n\n" 'SELECT event_id, status FROM __source("raw_events")\n' ) }, @@ -631,7 +631,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run **build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -643,7 +643,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run f"from python.{test_case.dependency_dir}.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" - " payload = ctx.result_of(node_function=prepare_events).payload\n" + " '''Test loader raw_events.'''\n payload = ctx.result_of(node_function=prepare_events).payload\n" " return [{'event_id': payload['event_id']}]\n" ), ), @@ -752,9 +752,9 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" - " - name: raw_rollup\n" + " - name: raw_rollup\n description: Test source raw_rollup.\n" " managed: true\n" ), loader_py=( @@ -763,15 +763,15 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'price_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_prices(ctx):\n" - " return [{'price_id': 1}]\n\n" + " '''Test loader fetch_prices.'''\n return [{'price_id': 1}]\n\n" "@loader(depends_on=[fetch_events, fetch_prices])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " prices = ctx.loader(fetch_prices)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination} UNION ALL " @@ -780,16 +780,16 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va " {'name': 'page_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_pages(ctx):\n" - " return [{'page_id': 1}]\n\n" + " '''Test loader fetch_pages.'''\n return [{'page_id': 1}]\n\n" "@loader(depends_on=[fetch_pages], write_strategy='table', " "columns=[{'name': 'page_id', 'type': 'INTEGER'}])\n" "def normalize_pages(ctx):\n" - " pages = ctx.loader(fetch_pages)\n" + " '''Test loader normalize_pages.'''\n pages = ctx.loader(fetch_pages)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT page_id FROM {pages.destination}')\n\n" "@loader(depends_on=[normalize_pages])\n" "def raw_rollup(ctx):\n" - " pages = ctx.loader(normalize_pages)\n" + " '''Test loader raw_rollup.'''\n pages = ctx.loader(normalize_pages)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT page_id FROM {pages.destination}')\n" ), @@ -879,7 +879,9 @@ def test_given_chained_loader_project_when_selecting_loader_then_expands_expecte tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" @@ -887,10 +889,10 @@ def test_given_chained_loader_project_when_selecting_loader_then_expands_expecte " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'load_seq': 1}, {'event_id': 2, 'load_seq': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'load_seq': 1}, {'event_id': 2, 'load_seq': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -949,17 +951,19 @@ def test_given_chained_loader_project_when_excluding_dependency_then_prunes_depe tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -1000,17 +1004,19 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(destination='custom_fetch_events', write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, '{events.table_name}' AS source_table " 'FROM {events.destination}")\n' @@ -1049,10 +1055,10 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'amount', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'amount': 100}, {'event_id': 2, 'amount': 200}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'amount': 100}, {'event_id': 2, 'amount': 200}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1077,7 +1083,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1088,7 +1094,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1109,7 +1115,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is not None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is not None:\n" " return [\n" " {'event_id': 2, 'amount': 250, 'load_seq': 1},\n" " {'event_id': 3, 'amount': 300, 'load_seq': 1},\n" @@ -1120,7 +1126,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1144,7 +1150,7 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1210,9 +1216,9 @@ def test_given_loader_dependency_failure_when_loading_then_only_dependents_skip( repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1222,15 +1228,15 @@ def test_given_loader_dependency_failure_when_loading_then_only_dependents_skip( " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ), ) @@ -1275,9 +1281,9 @@ def test_given_loader_dependency_failure_when_building_then_only_dependents_skip repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1287,23 +1293,23 @@ def test_given_loader_dependency_failure_when_building_then_only_dependents_skip " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ) | { "models/bad_model.sql": ( - 'MODEL (materialized table);\n\nSELECT event_id FROM __source("raw_bad")\n' + 'MODEL (description "Test model bad_model.", materialized table);\n\nSELECT event_id FROM __source("raw_bad")\n' ), "models/ok_model.sql": ( - 'MODEL (materialized table);\n\nSELECT event_id FROM __source("raw_ok")\n' + 'MODEL (description "Test model ok_model.", materialized table);\n\nSELECT event_id FROM __source("raw_ok")\n' ), }, ) @@ -1350,9 +1356,9 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1362,15 +1368,15 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ), ) @@ -1407,7 +1413,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_contract_events\n" + " - name: raw_contract_events\n description: Test source raw_contract_events.\n" " managed: true\n" " write_strategy: table\n" " contract: enforced\n" @@ -1419,7 +1425,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_contract_events(ctx):\n" - " return [{'event_id': 1, 'extra_note': 'not declared'}]\n" + " '''Test loader raw_contract_events.'''\n return [{'event_id': 1, 'extra_note': 'not declared'}]\n" ), ), ), @@ -1430,7 +1436,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_contract_events\n" + " - name: raw_contract_events\n description: Test source raw_contract_events.\n" " managed: true\n" " write_strategy: table\n" " contract: enforced\n" @@ -1444,7 +1450,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_contract_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_contract_events.'''\n return [{'event_id': 1}]\n" ), ), ), @@ -1455,7 +1461,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1469,7 +1475,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1, 'amount': 100}]\n" " return [{'event_id': 2, 'load_seq': 2, 'amount': 'one hundred'}]\n" ), @@ -1480,17 +1486,19 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ command=("--no-color", "load", "--select", "+raw_events"), expected_error_fragment="contract has extra columns: extra_note", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', contract='enforced', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'extra_note': 'not declared'}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'extra_note': 'not declared'}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -1501,15 +1509,17 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ command=("--no-color", "load", "--select", "+raw_events"), expected_error_fragment="returned no rows and has no destination declared", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def fetch_events(ctx):\n" - " ctx.execute_sql('SELECT 1')\n\n" + " '''Test loader fetch_events.'''\n ctx.execute_sql('SELECT 1')\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT * FROM {events.destination}")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py index 7e0da24b9e..1cd132172c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py @@ -16,17 +16,17 @@ _SOURCES: str = ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" ) _LOADERS: str = ( "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" "@loader(depends_on=[raw_regions])\n" "def raw_customers(ctx):\n" - " table = 'raw_' + 'regions'\n" + " '''Test loader raw_customers.'''\n table = 'raw_' + 'regions'\n" " ctx.query(f'SELECT count(*) FROM {table}')\n" " return [{'id': 1}]\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 26b32b58a4..d916c9b809 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -20,7 +20,10 @@ f'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{DATABASE_FILE}"\n' "\n[migrations]\nold_name_views = false\n" ) -_SOURCES_YML: str = "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" +_SOURCES_YML: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" +) def orders_sql(*, migrate_from: str) -> str: @@ -28,7 +31,7 @@ def orders_sql(*, migrate_from: str) -> str: migration: str = {"": ""}.get(migrate_from, f' migrate_from "{migrate_from}",\n') return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -136,7 +139,7 @@ def fct_orders_sql(*, columns: str, extra_config: str = "", replay: bool = True) """Return an incremental orders fact model projecting the given value columns.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -293,7 +296,7 @@ def old_name_model_sql( """Return a model of the given materialization over raw orders.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" f"{_OLD_NAME_HEADERS[materialized]}" f"{extra_config}" ");\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py index 4622bb60a0..fad925b216 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py @@ -44,7 +44,7 @@ "from sqlbuild.tasks import task\n\n\n" "@task\n" "def report(ctx):\n" - f' ctx.query("SELECT count(*) FROM {OLD_NAME_SCHEMA}.{OLD_NAME_ORIGIN}").fetchall()\n' + f' """Test task report."""\n ctx.query("SELECT count(*) FROM {OLD_NAME_SCHEMA}.{OLD_NAME_ORIGIN}").fetchall()\n' ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py index 66e3ac3042..d41d8bb80a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py @@ -45,7 +45,7 @@ def prepare_motherduck_build_project(*, tmp_path: Path) -> tuple[Path, str]: schema_name=schema_name, ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "SELECT 1 AS order_id, 'classic' AS waffle_name UNION ALL " "SELECT 2 AS order_id, 'liege' AS waffle_name" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 4a7dc4a4a7..6dd49631fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -18,14 +18,15 @@ f'[connection]\ndatabase = "{DIAMOND_DATABASE_FILE}"\n' ) _DIAMOND_SOURCES_YML: str = ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ) _DIAMOND_ROOT_PATH: str = "models/orders_0.sql" def _incremental_header(*, extra_config: str) -> str: return ( - "MODEL (\n" + "MODEL (description 'Test model orders_0.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -107,6 +108,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: _MISSING_SOURCE_TABLES_SOURCES_YML: str = """sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders freshness: @@ -114,6 +116,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: column: updated_at type: timestamp - name: raw_customers + description: Test source raw_customers. schema: raw table: customers freshness: @@ -121,6 +124,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: column: updated_at type: timestamp - name: raw_payments + description: Test source raw_payments. schema: raw table: payments freshness: @@ -145,13 +149,16 @@ def prepare_missing_source_tables_project(*, tmp_path: Path) -> Path: ), "sources/raw.yml": _MISSING_SOURCE_TABLES_SOURCES_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/customers.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_customers")\n' + 'MODEL (description "Test model customers.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_customers")\n' ), "models/payments.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_payments")\n' + 'MODEL (description "Test model payments.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_payments")\n' ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py index 21e1406eb4..d27385997e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py @@ -31,7 +31,7 @@ "error[S405]", "1 source table read by selected resources does not exist in the warehouse", "source 'raw_payments' (raw.payments), read by payments", - "'raw_payments' at sources/raw.yml:16", + "'raw_payments' at sources/raw.yml:18", ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py index 5a90a99e36..412e4bfbb8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py @@ -30,14 +30,14 @@ def test_given_physical_source_shape_when_rebinding_then_rejects_downstream_befo ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n table: raw_orders\n schema: main\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: raw_orders\n schema: main\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized table, schema analytics); SELECT * FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table, schema analytics); SELECT * FROM __source("raw_orders")' ) (tmp_path / "models/report.sql").write_text( - 'MODEL (materialized table, schema analytics); SELECT missing FROM __ref("orders")' + 'MODEL (description "Test model report.", materialized table, schema analytics); SELECT missing FROM __ref("orders")' ) result: subprocess.CompletedProcess[str] = run_sqb( command=test_case.command, project_dir=tmp_path diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py index 783b12cb8b..c0b04aa5ca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py @@ -95,7 +95,7 @@ def test_given_removed_interface_when_running_cli_then_reports_unknown_argument( ), RemovedConfigTestCase( "removed unchanged MODEL field", - "MODEL (materialized table, run_despite_unchanged always); SELECT 1 AS order_id", + "MODEL (description 'Test model.', materialized table, run_despite_unchanged always); SELECT 1 AS order_id", "run_despite_unchanged", "models/orders.sql", "", @@ -158,7 +158,7 @@ def test_given_removed_config_when_running_cli_then_explains_direct_mode( ) -> None: files: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (materialized view);\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\nSELECT 1 AS order_id", } files[test_case.filename] = test_case.prefix + test_case.content project: Path = prepare_inline_project( @@ -193,7 +193,7 @@ def test_given_prepare_version_hook_when_running_cli_then_rejects_removed_hook( project_name="removed_hook", repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "models/orders.sql": "MODEL (materialized custom_order);\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized custom_order);\nSELECT 1 AS order_id", "materializations/custom_order.py": ( "from sqlbuild.executor.custom.models import PrepareVersionContext\n\n" "def prepare_version(ctx: PrepareVersionContext):\n pass\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index 56c6e93818..b37f33056d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -471,7 +471,7 @@ def orders_sql(*, migrate_from: str = "", where: str = "") -> str: migrate_from, f' migrate_from "{migrate_from}",\n migrate_force true,\n' ) return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -488,7 +488,10 @@ def orders_sql(*, migrate_from: str = "", where: str = "") -> str: def report_view_sql() -> str: """Return a project view that reads the destination by name.""" - return f'MODEL (materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + return ( + 'MODEL (description "Test model.", ' + f'materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + ) def write_migration_project( @@ -507,7 +510,8 @@ def write_migration_project( ) + _JANITOR_CONFIG, "sources/raw.yml": ( - f"sources:\n - name: raw_orders\n schema: {raw_schema_name}\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + f" schema: {raw_schema_name}\n" " table: raw_orders\n" ), } @@ -636,7 +640,7 @@ def fact_orders_sql(*, amount: str = "amount_cents", columns: str = "") -> str: declarations: str = {"": ""}.get(columns, f" columns ({columns}),\n") return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -717,7 +721,7 @@ def old_name_model_sql( migration: str = {"": ""}.get(migrate_from, f" migrate_from {migrate_from},\n") return ( - f"MODEL (\n{_OLD_NAME_HEADERS[materialized]}{migration});\n\n" + f"MODEL (description 'Test model.',\n{_OLD_NAME_HEADERS[materialized]}{migration});\n\n" f'SELECT {columns} FROM __source("raw_orders")\n' ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py index 782d927e5c..f02490f61e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py @@ -68,11 +68,12 @@ def test_given_dlt_source_when_loading_to_postgres_then_table_is_materialized_an f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 1284767841..8cdd464dc8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -262,14 +262,14 @@ def test_given_python_result_when_running_check_on_postgres_then_persists_node_r "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -390,7 +390,7 @@ def test_given_source_deferral_env_when_building_on_postgres_then_reads_prod_and ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -403,10 +403,10 @@ def test_given_source_deferral_env_when_building_on_postgres_then_reads_prod_and "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -485,7 +485,7 @@ def test_given_partial_source_type_enforcement_when_building_on_postgres_then_ca ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {schema_name}\n" " type_enforcement: true\n" " columns:\n" @@ -493,7 +493,7 @@ def test_given_partial_source_type_enforcement_when_building_on_postgres_then_ca " type: INTEGER\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT order_id, status, amount_cents FROM __source("raw_orders")' ), }, @@ -709,7 +709,7 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write PostgresScenarioLocalReplayE2ETestCase( description="captures postgres fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " DATE_TRUNC('day', event_ts) AS event_day,\n" @@ -720,7 +720,7 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write "GROUP BY customer_id, DATE_TRUNC('day', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, CAST('2026-01-01 08:15:00' AS TIMESTAMP)" @@ -751,12 +751,12 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write description="reports postgres local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -855,7 +855,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -870,10 +870,10 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -933,7 +933,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -942,7 +942,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -968,7 +968,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -979,7 +979,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1000,7 +1000,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1011,7 +1011,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1037,7 +1037,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_postgres_then repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index b2242b36e8..cca5fb17fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -63,6 +63,7 @@ def test_given_provider_exporter_when_building_then_receives_redacted_events_bef from sqlbuild.providers import Provider class EventSink(Provider): + '''Test provider EventSink.''' path: Path | None = None def setup(self, ctx): @@ -106,7 +107,7 @@ def failing_exporter(event: LifecycleEvent, event_sink: EventSink) -> None: + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model orders.", materialized table); SELECT 'do-not-export-this-sql' AS secret_value """ @@ -191,7 +192,7 @@ def export_output(record: CommandOutputRecord) -> None: """ ).strip() + "\n", - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, ) @@ -237,7 +238,7 @@ def test_given_configured_provider_and_sink_when_lineage_runs_then_skips_extensi "from pathlib import Path\n" "from sqlbuild.providers import Provider\n" "class OutputProvider(Provider):\n" - " def setup(self, ctx):\n" + " '''Test provider OutputProvider.'''\n def setup(self, ctx):\n" " del ctx\n" f" Path({str(provider_marker)!r}).write_text('initialized', encoding='utf-8')\n" " def write(self, record):\n" @@ -252,7 +253,7 @@ def test_given_configured_provider_and_sink_when_lineage_runs_then_skips_extensi "def export_output(record, output_provider: OutputProvider):\n" " output_provider.write(record)\n" ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, ) @@ -303,7 +304,7 @@ def test_given_no_exporters_and_exploding_provider_when_non_provider_command_run f"Path({str(helper_marker_path)!r}).write_text('imported', encoding='utf-8')\n" "def encode(value):\n return value\n" ), - "models/example.sql": "MODEL (materialized view);\n\nSELECT 1 AS value\n", + "models/example.sql": "MODEL (description 'Test model example.', materialized view);\n\nSELECT 1 AS value\n", }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 0ec54603f1..1836ed8fa5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -53,6 +53,7 @@ class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' marker_path: str = Field(validation_alias="MARKER_PATH") def setup(self, ctx): @@ -124,6 +125,7 @@ def test_given_provider_usages_when_planning_then_text_and_json_include_selected class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "plan-output" """ ).strip() @@ -136,6 +138,7 @@ class MarkerProvider(Provider): @loader def raw_orders(marker_provider: MarkerProvider): + '''Test loader raw_orders.''' return [{"order_id": 1, "amount": 10}] """ ).strip() @@ -148,6 +151,7 @@ def raw_orders(marker_provider: MarkerProvider): @task def publish_orders(marker_provider: MarkerProvider): + '''Test task publish_orders.''' marker_provider.label """ ).strip() @@ -160,6 +164,7 @@ def publish_orders(marker_provider: MarkerProvider): @hook def mark_pre(marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.label """ ).strip() @@ -187,6 +192,7 @@ def materialize( """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -199,7 +205,7 @@ def materialize( + "\n", "models/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized copy_table, pre_hooks [python("mark_pre")] ); @@ -300,6 +306,7 @@ def test_given_project_with_provider_when_running_commands_then_provider_is_inje class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' marker_path: str = Field(validation_alias="MARKER_PATH") label: str = "provider-e2e" @@ -324,6 +331,7 @@ def mark(self, value): @task def provider_task(ctx, marker_provider: MarkerProvider): + '''Test task provider_task.''' marker_provider.mark(f"run_task:{marker_provider.label}") """ ).strip() @@ -336,6 +344,7 @@ def provider_task(ctx, marker_provider: MarkerProvider): @asset def provider_asset(ctx, marker_provider: MarkerProvider): + '''Test asset provider_asset.''' marker_provider.mark(f"build_asset:{marker_provider.label}") """ ).strip() @@ -348,6 +357,7 @@ def provider_asset(ctx, marker_provider: MarkerProvider): @check(depends_on=()) def provider_check(ctx, marker_provider: MarkerProvider): + '''Test check provider_check.''' marker_provider.mark(f"check:{marker_provider.label}") return True """ @@ -361,6 +371,7 @@ def provider_check(ctx, marker_provider: MarkerProvider): @loader def raw_provider_events(ctx, marker_provider: MarkerProvider): + '''Test loader raw_provider_events.''' marker_provider.mark(f"load:{marker_provider.label}") return [{"event_id": 1, "label": marker_provider.label}] """ @@ -370,6 +381,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): """ sources: - name: raw_provider_events + description: Test source raw_provider_events. managed: true write_strategy: table columns: @@ -441,6 +453,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): @task def failing_task(ctx, marker_provider: MarkerProvider): + '''Test task failing_task.''' marker_provider.mark("task") raise RuntimeError("intentional provider failure") """ @@ -463,6 +476,7 @@ def failing_task(ctx, marker_provider: MarkerProvider): @asset def failing_asset(ctx, marker_provider: MarkerProvider): + '''Test asset failing_asset.''' marker_provider.mark("asset") raise RuntimeError("intentional provider failure") """ @@ -485,6 +499,7 @@ def failing_asset(ctx, marker_provider: MarkerProvider): @check(depends_on=()) def failing_check(ctx, marker_provider: MarkerProvider): + '''Test check failing_check.''' marker_provider.mark("check") raise RuntimeError("intentional provider failure") """ @@ -507,6 +522,7 @@ def failing_check(ctx, marker_provider: MarkerProvider): @loader def raw_provider_events(ctx, marker_provider: MarkerProvider): + '''Test loader raw_provider_events.''' marker_provider.mark("load") raise RuntimeError("intentional provider failure") """ @@ -516,6 +532,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): """ sources: - name: raw_provider_events + description: Test source raw_provider_events. managed: true write_strategy: table columns: @@ -587,11 +604,12 @@ def test_given_provider_project_when_running_compile_or_plan_then_provider_setup @task def provider_task(ctx, marker_provider: MarkerProvider): + '''Test task provider_task.''' marker_provider.mark("task") """ ).strip() + "\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, ) @@ -648,6 +666,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ @task def alias_task(ctx, marker_provider: AliasMarkerProvider): + '''Test task alias_task.''' marker_provider.mark("task") """ ).strip() @@ -704,7 +723,7 @@ def materialize( """ ).strip() + "\n", - "models/orders.sql": "MODEL (materialized marker_mat);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized marker_mat);\n\nSELECT 1 AS id\n", }, ) @@ -759,6 +778,7 @@ def test_given_concurrent_provider_backed_nodes_when_running_command_then_share_ class ConcurrentMarkerProvider(Provider): + '''Test provider ConcurrentMarkerProvider.''' marker_path: str = {str(marker_path)!r} @property @@ -786,12 +806,14 @@ def mark(self, label): @loader def raw_alpha(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_alpha.''' concurrent_marker_provider.mark("alpha") return [{"event_id": 1}] @loader def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_beta.''' concurrent_marker_provider.mark("beta") return [{"event_id": 2}] """ @@ -801,12 +823,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): """ sources: - name: raw_alpha + description: Test source raw_alpha. managed: true write_strategy: table columns: - name: event_id type: INTEGER - name: raw_beta + description: Test source raw_beta. managed: true write_strategy: table columns: @@ -816,10 +840,10 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): ).strip() + "\n", "models/fact_alpha.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' + 'MODEL (description "Test model fact_alpha.", materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' ), "models/fact_beta.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_beta")\n' + 'MODEL (description "Test model fact_beta.", materialized table);\n\nSELECT * FROM __source("raw_beta")\n' ), }, ) @@ -880,12 +904,14 @@ def test_given_python_hooks_with_provider_when_building_then_hooks_use_provider_ @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' ctx.providers.marker_provider.mark("pre_ctx") marker_provider.mark("pre_injected") @hook def mark_post(hook_context, marker_provider: MarkerProvider): + '''Test hook mark_post.''' hook_context.providers["marker_provider"].mark("post_ctx") marker_provider.mark("post_injected") """ @@ -893,7 +919,7 @@ def mark_post(hook_context, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")], post_hooks [python("mark_post")] @@ -948,18 +974,20 @@ def test_given_view_model_python_hooks_with_provider_when_building_then_hooks_us @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("view_pre") @hook def mark_post(ctx, marker_provider: MarkerProvider): + '''Test hook mark_post.''' marker_provider.mark("view_post") """ ).strip() + "\n", "models/orders_view.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_view.", materialized view, pre_hooks [python("mark_pre")], post_hooks [python("mark_post")] @@ -992,7 +1020,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/incremental_orders.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model incremental_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -1020,7 +1048,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/hourly_activity.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model hourly_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -1046,7 +1074,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): extra_repo_files={ "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT 1 AS order_id, TIMESTAMP '2026-01-01 00:00:00' AS ordered_at """ @@ -1067,7 +1095,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/customer_snapshot.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1096,7 +1124,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/custom_orders.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model custom_orders.", materialized copy_table, pre_hooks [python("mark_hook")], post_hooks [python("mark_hook")] @@ -1149,6 +1177,7 @@ def test_given_materialization_hooks_with_provider_when_building_then_hooks_use_ @hook def mark_hook(ctx, marker_provider: MarkerProvider): + '''Test hook mark_hook.''' marker_provider.mark(f"{ctx.model_name}:{ctx.phase}") """ ).strip() @@ -1203,13 +1232,14 @@ def test_given_untyped_python_hook_provider_parameter_when_building_then_provide @hook def mark_pre(ctx, marker_provider): + '''Test hook mark_pre.''' marker_provider.mark("untyped") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1263,6 +1293,7 @@ def test_given_python_hook_with_provider_when_hook_fails_then_provider_tears_dow @hook def failing_pre_hook(ctx, marker_provider: MarkerProvider): + '''Test hook failing_pre_hook.''' marker_provider.mark("pre") raise RuntimeError("intentional provider hook failure") """ @@ -1270,7 +1301,7 @@ def failing_pre_hook(ctx, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("failing_pre_hook")] ); @@ -1324,6 +1355,7 @@ def test_given_python_post_hook_with_provider_when_hook_fails_then_provider_tear @hook def failing_post_hook(ctx, marker_provider: MarkerProvider): + '''Test hook failing_post_hook.''' marker_provider.mark("post") raise RuntimeError("intentional provider post hook failure") """ @@ -1331,7 +1363,7 @@ def failing_post_hook(ctx, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [python("failing_post_hook")] ); @@ -1395,13 +1427,14 @@ def test_given_alias_imported_provider_annotation_on_hook_when_building_then_cli @hook def mark_pre(ctx, marker_provider: AliasMarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1451,6 +1484,7 @@ def test_given_provider_annotation_mismatch_on_hook_when_building_then_cli_print class OtherProvider(Provider): + '''Test provider OtherProvider.''' pass """ ).strip() @@ -1463,13 +1497,14 @@ class OtherProvider(Provider): @hook def mark_pre(ctx, marker_provider: OtherProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1524,13 +1559,14 @@ def test_given_python_hook_kwarg_matches_provider_when_building_then_cli_prints_ @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre", marker_provider: "literal")] ); @@ -1609,13 +1645,14 @@ def test_given_python_hook_kwarg_matches_context_when_building_then_cli_prints_c @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre", {test_case.context_parameter_name}: "literal")] ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 78b1284e7a..7cc11f4498 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -34,7 +34,7 @@ def project_toml(*, target_settings: str = "") -> str: _PROJECT_TOML: str = project_toml() _SOURCES_YML: str = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -44,17 +44,17 @@ def project_toml(*, target_settings: str = "") -> str: " - name: order_date\n" ) STG_ORDERS: str = ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model.',\n materialized view,\n);\n\n" "SELECT\n order_id,\n customer_id,\n amount,\n order_date\n" 'FROM __source("raw_orders")\n' ) FACT_ORDERS: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n o.order_id,\n o.customer_id,\n o.amount,\n o.order_date\n" 'FROM __ref("stg_orders") AS o\n' ) CUSTOMER_TOTALS: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n customer_id,\n SUM(amount) AS total_amount\n" 'FROM __ref("fact_orders")\nGROUP BY customer_id\n' ) @@ -81,20 +81,21 @@ def project_toml(*, target_settings: str = "") -> str: } UDF_FILE: tuple[str, str] = ( "functions/sql/udf__is_large_order.sql", - "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", + "FUNCTION (description 'Test function udf__is_large_order.'," + "\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", ) CENTS_MACRO: dict[str, str] = { "models/staging/_sqlbuild/_macros/cents.py": ( 'def to_cents(expression: str) -> str:\n return f"({expression}) * 100"\n' ), "models/staging/stg_order_cents.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_cents.',\n materialized view,\n);\n\n" 'SELECT\n order_id,\n @to_cents("amount") AS amount_cents\nFROM __ref("stg_orders")\n' ), } ORDER_REFUNDS: dict[str, str] = { "models/staging/stg_order_refunds.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_refunds.',\n materialized view,\n);\n\n" 'SELECT\n order_id,\n @to_cents("amount") AS refund_cents\nFROM __ref("stg_orders")\n' ), } @@ -115,26 +116,28 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: return { "models/marts/order_amounts.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model order_amounts.',\n materialized view,\n);\n\n" f'SELECT amount FROM __ref("{first}")\nUNION ALL\n' f'SELECT amount FROM __ref("{second}")\n' ), "models/marts/order_amount_reads.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' + 'MODEL (description "Test model order_amount_reads.",' + '\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' ), } ORDER_EXPORT: dict[str, str] = { "models/marts/order_export.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' + 'MODEL (description "Test model order_export.",' + '\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' ), } CUSTOMER_DECLARATIONS: dict[str, str] = { "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", "seeds/customers.yml": ( "seeds:\n" - " - name: customers\n" + " - name: customers\n description: Test seed customers.\n" " columns:\n" " - name: customer_id\n" " type: INTEGER\n" @@ -147,7 +150,7 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: ), "sources/customers.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" ' expression: "SELECT 10 AS customer_id UNION ALL SELECT 11"\n' " columns:\n" " - name: customer_id\n" @@ -157,7 +160,8 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: } ORDER_SHAPE: dict[str, str] = { "models/marts/_sqlbuild/_schemas/order_shape.sql": ( - "SCHEMA (\n name order_shape,\n columns (\n customer_id (audits [relationships " + "SCHEMA (description 'Test schema.'," + "\n name order_shape,\n columns (\n customer_id (audits [relationships " '(to __ref("stg_orders"), field customer_id)]),\n ),\n);\n' ), "models/marts/fact_orders.sql": FACT_ORDERS.replace( @@ -170,7 +174,7 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: "def staged_orders() -> str:\n return '__ref(\"stg_orders\")'\n" ), "models/staging/stg_order_count.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_count.',\n materialized view,\n);\n\n" "SELECT COUNT(*) AS order_count\nFROM @staged_orders()\n" ), } @@ -203,7 +207,8 @@ def order_history_files(*, materialized: str, udf: bool) -> dict[str, str]: return { "models/marts/order_history.sql": ( - f"MODEL (\n{_HISTORY_HEADERS[materialized]});\n\n" + "MODEL (description 'Test model order_history.'," + f"\n{_HISTORY_HEADERS[materialized]});\n\n" f"SELECT\n order_id,\n amount,\n order_date{_UDF_COLUMN[udf]}\n" 'FROM __ref("stg_orders")\n' ), @@ -278,7 +283,7 @@ def declare_missing_column_origin(*, project_dir: Path) -> None: contents: str = path.read_text(encoding="utf-8") _ = path.write_text( contents.replace( - "MODEL (\n", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n", 1 + "MODEL (", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n ", 1 ).replace(" order_date\n", " order_date,\n amount AS revenue\n", 1), encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 8e722ff862..80fa105a91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -22,7 +22,9 @@ def write_unevaluated_rules_project( adapter: str = "duckdb", selected_rules: tuple[str, ...] = ("SQBRSQL035",), resource_path: str = "models/staging/orders.sql", - resource_template: str = "MODEL ({options});\nSELECT {expression} AS order_id", + resource_template: str = ( + "MODEL (description 'Test model orders.', {options});\nSELECT {expression} AS order_id" + ), extra_files: tuple[tuple[str, str], ...] = (), ) -> None: (project_dir / "sqlbuild_project.toml").write_text( @@ -31,7 +33,9 @@ def write_unevaluated_rules_project( ) models: Path = project_dir / "models/staging" models.mkdir(parents=True) - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n" + ) relative_path: str contents: str for relative_path, contents in extra_files: @@ -240,13 +244,14 @@ def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, st "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" ), "models/staging/stg_orders.sql": ( - "MODEL (\n materialized view,\n contract enforced,\n columns (\n" + "MODEL (description 'Test model stg_orders.',\n materialized view,\n" + " contract enforced,\n columns (\n" " order_id (type INTEGER, nullable false, audits [not_null]),\n" " amount (type DOUBLE),\n ),\n);\n\n" "SELECT CAST(1 AS INTEGER) AS order_id, CAST(10.5 AS DOUBLE) AS amount\n" ), "models/marts/order_totals.sql": ( - "MODEL (\n columns (\n" + "MODEL (description 'Test model order_totals.',\n columns (\n" " order_id (type INTEGER, nullable false, audits [not_null]),\n" " net_amount (type DOUBLE),\n ),\n);\n\n" "SELECT\n o.order_id,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py index a66ace2c1b..9a4faa1789 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py @@ -65,7 +65,7 @@ lambda root: write_project_file( root, "models/marts/order_counts.sql", - 'MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n', + 'MODEL (description "Test model order_counts.");\n\nSELECT order_id FROM __ref("stg_orders")\n', ), ), RulesCacheEditCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py index 2efe6d27b9..ede09a9dba 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -89,7 +89,7 @@ ) _KEYED_SOURCE_YML: str = ( "sources:\n" - " - name: landing_orders\n" + " - name: landing_orders\n description: Test source landing_orders.\n" " schema: landing\n" " columns:\n" " - name: order_id\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index 2d169f4577..41516c64ab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -25,7 +25,7 @@ UnevaluatedResourceCase( "model guard", "models/staging/orders.sql", - "MODEL ();\nSELECT {expression} AS order_id", + "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", 0, ), UnevaluatedResourceCase( @@ -34,12 +34,17 @@ "AUDIT ();\nSELECT {expression} AS order_id " 'FROM __ref("orders") JOIN __ref("customers") USING (order_id)', expected_evaluated_models=2, - extra_files=(("models/staging/customers.sql", "MODEL ();\nSELECT 1 AS order_id\n"),), + extra_files=( + ( + "models/staging/customers.sql", + "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", + ), + ), ), UnevaluatedResourceCase( "model the compiler accepts but the Rules parser rejects", "models/staging/orders.sql", - "MODEL ();\nWITH items AS (SELECT [1, 2] AS quantities)\n" + "MODEL (description 'Test model orders.');\nWITH items AS (SELECT [1, 2] AS quantities)\n" "SELECT 1 AS order_id, list_transform(i.quantities, lambda q: q + 1) AS shipped\n" "FROM items AS i", expected_evaluated_models=0, @@ -120,7 +125,8 @@ def test_given_prefix_not_like_when_running_rules_then_evaluates_the_complete_mo write_unevaluated_rules_project( project_dir=tmp_path, resource_path="models/staging/orders.sql", - resource_template="MODEL (database warehouse, schema analytics);\n" + test_case.sql, + resource_template="MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql, adapter="snowflake", ) for _ in range(2): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py index 6461358d51..7a7e88507a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py @@ -22,7 +22,7 @@ files={ "functions/sql/customer_orders.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (customer_id INTEGER, order_id INTEGER) ); @@ -33,7 +33,7 @@ + "\n", "models/order_summary.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model order_summary.", materialized table); WITH customer_ids AS ( SELECT 7 AS customer_id @@ -58,7 +58,7 @@ description="generic audit argument may reference its local base CTE", files={ "models/orders.sql": ( - "MODEL (\n materialized table,\n audits [\n order_group_rate (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n audits [\n order_group_rate (\n" ' evaluation_sql "SELECT 1 AS is_ok FROM base",\n' " thresholds (error (below 0.5)),\n" " ),\n ],\n);\n\nSELECT 1 AS order_id\n" @@ -93,7 +93,7 @@ description="opaque audit SQL does not hide a later dead CTE", files={ "models/orders.sql": ( - "MODEL (\n materialized table,\n audits [\n order_group_rate (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n audits [\n order_group_rate (\n" ' evaluation_sql "SELECT 1 AS is_ok FROM base",\n' " thresholds (error (below 0.5)),\n" " ),\n ],\n);\n\nSELECT 1 AS order_id\n" @@ -129,10 +129,10 @@ description="test harness output keeps its helper CTE reachable", files={ "models/raw_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 7 AS order_id, 25 AS amount\n" + "MODEL (description 'Test model raw_orders.', materialized table);\n\nSELECT 7 AS order_id, 25 AS amount\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __ref("raw_orders")\n' ), "tests/unit/orders.sql": dedent( """ @@ -157,7 +157,7 @@ description="ordinary audit parameters do not hide a dead CTE", files={ "models/orders.sql": ( - "MODEL (materialized table, audits [order_rows]);\n\nSELECT 1 AS order_id\n" + "MODEL (description 'Test model orders.', materialized table, audits [order_rows]);\n\nSELECT 1 AS order_id\n" ), "audits/generic/order_rows.sql": dedent( """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 2ff0dbebef..77f6d2d840 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -77,17 +77,18 @@ def build_scenario_project_files() -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT\n" " id AS order_id,\n" " amount\n" 'FROM __source("raw_orders")\n' ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" "SELECT\n" " SUM(amount) AS total_amount\n" 'FROM __ref("orders")\n' @@ -106,7 +107,7 @@ def build_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), "tests/scenarios/nested/orders_assert_pass.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario orders_assert_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -117,7 +118,7 @@ def build_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), "tests/scenarios/order_totals_fail.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario order_totals_fail.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -142,16 +143,19 @@ def build_partial_fixture_scenario_project_files() -> dict[str, str]: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" " - name: status\n type: VARCHAR\n" ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __source("raw_orders")\n' + ), "tests/scenarios/partial_orders.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario partial_orders.');\n\n" "WITH\n" "__source__raw_orders AS (SELECT 1 AS order_id),\n" "__expected__orders AS (\n" @@ -170,13 +174,14 @@ def build_scenario_python_hooks_project_files() -> dict[str, str]: "from sqlbuild.hooks import hook\n\n" "@hook\n" "def log_scenario_model(ctx):\n" - " ctx.execute_sql(\n" + " '''Test hook log_scenario_model.'''\n ctx.execute_sql(\n" ' f"CREATE TABLE {ctx.destination.schema}.scenario_hook_log AS "\n' " f\"SELECT '{ctx.model_name}' AS model_name, '{ctx.phase}' AS phase\"\n" " )\n" ) repo_files["models/orders.sql"] = ( - 'MODEL (materialized table, post_hooks [python("log_scenario_model")]);\n\n' + 'MODEL (description "Test model orders.", ' + 'materialized table, post_hooks [python("log_scenario_model")]);\n\n' "SELECT\n" " id AS order_id,\n" " amount\n" @@ -209,7 +214,8 @@ def build_real_warehouse_local_replay_project_files( return { "sqlbuild_project.toml": project_toml, "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: raw\n table: raw_events\n" ), "models/event_rollup.sql": model_sql, f"tests/scenarios/{scenario_name}.sql": scenario_sql, @@ -222,16 +228,17 @@ def build_real_warehouse_remote_scenario_project_files(*, project_toml: str) -> return { "sqlbuild_project.toml": project_toml, "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: raw\n table: raw_events\n" ), "models/stg_events.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_events.', materialized view);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' "WHERE amount_cents >= 1000\n" ), "models/event_rollup.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model event_rollup.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " SUM(amount_cents) AS large_amount_cents,\n" @@ -240,7 +247,7 @@ def build_real_warehouse_remote_scenario_project_files(*, project_toml: str) -> "GROUP BY customer_id\n" ), "tests/scenarios/remote_event_rollup.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario remote_event_rollup.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py index 7c5aa935ef..6d4f51e748 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py @@ -26,7 +26,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB integer division in fixture, expected and assertion SQL", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 21 // 2 AS amount -- integer division\n" @@ -46,7 +46,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB escape strings between relation markers in assertion SQL", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -66,7 +66,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB escape string before the first relation marker", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -110,7 +110,7 @@ def test_given_dialect_specific_scenario_sql_when_testing_then_runs_authored_sql ScenarioUnresolvableCheckSqlE2ETestCase( description="DuckDB backslash before a quote leaves the assertion string unclosed", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -175,7 +175,7 @@ def test_given_project_dialect_scenario_sql_when_replaying_locally_then_resolves del repo_files["tests/scenarios/order_totals_fail.sql"] del repo_files["tests/scenarios/nested/orders_assert_pass.sql"] repo_files[f"tests/scenarios/{test_case.scenario_name}.sql"] = ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario orders_assert_pass.');\n\n" "WITH\n" f"__source__raw_orders AS (\n {test_case.source_fixture_sql}\n),\n" f"__assert__no_named_totals AS (\n {test_case.assertion_sql}\n)\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index e1234511e1..64c8fdf399 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -59,7 +59,9 @@ def test_given_unclosed_scenario_comment_when_running_cli_then_error_names_scena tmp_path: Path, ) -> None: repo_files: dict[str, str] = build_scenario_project_files() - repo_files["tests/scenarios/order_totals_pass.sql"] = "SCENARIO ();\nWITH /* unfinished" + repo_files["tests/scenarios/order_totals_pass.sql"] = ( + "SCENARIO (description 'Test scenario order_totals_pass.');\nWITH /* unfinished" + ) project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="scenario_comment", repo_files=repo_files ) @@ -192,13 +194,13 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "models/local_model_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_model_error.', materialized table);\n\n" "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_model_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_model_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -228,17 +230,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/is_large_order.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -276,17 +278,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/bad_sql_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", + "FUNCTION (description 'Test function bad_sql_function.', arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", ), ( "models/local_sql_function_setup_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_setup_error.', materialized table);\n\n" 'SELECT id, __udf("bad_sql_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_setup_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_setup_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -331,17 +333,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/bad_transpile_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function bad_transpile_function.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_transpile_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_transpile_error.', materialized table);\n\n" 'SELECT id, __udf("bad_transpile_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_transpile_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_transpile_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -374,18 +376,18 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/sql_runtime_error.sql", - "FUNCTION (arguments (amount INTEGER), returns INTEGER);\n\n" + "FUNCTION (description 'Test function sql_runtime_error.', arguments (amount INTEGER), returns INTEGER);\n\n" "CAST('bad' AS INTEGER)\n", ), ( "models/local_sql_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("sql_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -422,17 +424,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -477,17 +479,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' schema="analytics",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_error.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -538,17 +540,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> int:\n" - ' raise ValueError("python udf exploded")\n', + ' """Test function main."""\n raise ValueError("python udf exploded")\n', ), ( "models/local_python_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("python_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -900,13 +902,13 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "models/local_model_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_model_error.', materialized table);\n\n" "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_model_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_model_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -936,17 +938,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/is_large_order.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -984,17 +986,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/bad_sql_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", + "FUNCTION (description 'Test function bad_sql_function.', arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", ), ( "models/local_sql_function_setup_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_setup_error.', materialized table);\n\n" 'SELECT id, __udf("bad_sql_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_setup_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_setup_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1039,17 +1041,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/bad_transpile_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function bad_transpile_function.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_transpile_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_transpile_error.', materialized table);\n\n" 'SELECT id, __udf("bad_transpile_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_transpile_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_transpile_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1093,18 +1095,18 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/sql_runtime_error.sql", - "FUNCTION (arguments (amount INTEGER), returns INTEGER);\n\n" + "FUNCTION (description 'Test function sql_runtime_error.', arguments (amount INTEGER), returns INTEGER);\n\n" "CAST('bad' AS INTEGER)\n", ), ( "models/local_sql_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("sql_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1141,17 +1143,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1196,17 +1198,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' schema="analytics",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_error.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1257,17 +1259,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> int:\n" - ' raise ValueError("python udf exploded")\n', + ' """Test function main."""\n raise ValueError("python udf exploded")\n', ), ( "models/local_python_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("python_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index fbfed4847a..e2af4ae838 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -78,7 +78,7 @@ def write_scope_performance_project( def _write_scope_model(*, model_dir: Path, model_index: int, macro_name: str) -> None: (model_dir / f"model_{model_index:05d}.sql").write_text( - f'MODEL();\nSELECT @{macro_name}("id") AS id\n', encoding="utf-8" + f'MODEL(description "Test model.");\nSELECT @{macro_name}("id") AS id\n', encoding="utf-8" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py index fc2717e3dd..b61c1428bb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -24,7 +24,7 @@ def test_given_minimal_project_when_running_scope_aliases_then_outputs_are_offli 'name = "scope_e2e"\nadapter = "duckdb"\n', encoding="utf-8" ) (project_dir / "models" / "orders.sql").write_text( - "MODEL();\nSELECT 1 AS id\n", encoding="utf-8" + "MODEL(description 'Test model orders.');\nSELECT 1 AS id\n", encoding="utf-8" ) first: subprocess.CompletedProcess[str] = run_scope_alias( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py index 5e1b5ac518..fdfdc81dd2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py @@ -104,7 +104,7 @@ def test_given_empty_typed_seed_fields_when_loading_then_persists_nulls( ), "seeds/schema.yml": ( "seeds:\n" - " - name: nullable_mappings\n" + " - name: nullable_mappings\n description: Test seed nullable_mappings.\n" " columns:\n" " - name: mapping_id\n" " type: INTEGER\n" @@ -161,11 +161,11 @@ def test_given_standalone_seeds_when_writing_json_file_then_assets_are_complete_ ), "seeds/schema.yml": ( "seeds:\n" - " - name: first_seed\n" + " - name: first_seed\n description: Test seed first_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" - " - name: second_seed\n" + " - name: second_seed\n description: Test seed second_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -225,7 +225,7 @@ def test_given_failing_standalone_seed_when_writing_json_file_then_failure_is_ho ), "seeds/schema.yml": ( "seeds:\n" - " - name: invalid_seed\n" + " - name: invalid_seed\n description: Test seed invalid_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py index 21fbabdc1d..4faf5f6b84 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py @@ -252,7 +252,8 @@ def prepare_snowflake_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: staging_dir: Path = models_dir / "staging" staging_dir.mkdir(parents=True, exist_ok=True) (staging_dir / "stg_orders.sql").write_text( - "MODEL (materialized table, unique_key [order_id]);\n\n" + "MODEL (description 'Test model stg_orders.', " + "materialized table, unique_key [order_id]);\n\n" "SELECT * FROM (" "SELECT 1 AS order_id, 1 AS customer_id, 100 AS amount_cents UNION ALL " "SELECT 2 AS order_id, 2 AS customer_id, 200 AS amount_cents" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py index 1a22fe110e..1b47a05b6a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py @@ -65,11 +65,12 @@ def test_given_dlt_source_when_loading_to_snowflake_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 07c685357e..63f13f6223 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -181,14 +181,14 @@ def test_given_python_result_when_running_check_on_snowflake_then_persists_node_ "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -262,12 +262,12 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {schema_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -393,7 +393,7 @@ def test_given_source_deferral_env_when_building_on_snowflake_then_reads_prod_an "sqlbuild_local.toml": build_snowflake_local_config(schema_name=dev_schema_name), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -406,10 +406,10 @@ def test_given_source_deferral_env_when_building_on_snowflake_then_reads_prod_an "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -544,7 +544,7 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s SnowflakeScenarioLocalReplayE2ETestCase( description="captures snowflake fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " DATE_TRUNC('DAY', event_ts) AS event_day,\n" @@ -554,7 +554,7 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s "GROUP BY customer_id, DATE_TRUNC('DAY', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TO_TIMESTAMP_NTZ('2026-01-01 08:15:00') " @@ -585,12 +585,12 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s description="reports snowflake local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -613,14 +613,14 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s description="reports snowflake local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(amount_cents) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -1177,7 +1177,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_snowflake_t repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1191,7 +1191,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_snowflake_t "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -1256,7 +1256,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1271,10 +1271,10 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1329,7 +1329,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1338,7 +1338,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1364,7 +1364,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1375,7 +1375,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1396,7 +1396,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1407,7 +1407,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1432,7 +1432,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_snowflake_the repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py index d926ca1c2d..c500293d0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py @@ -69,11 +69,12 @@ def test_given_dlt_source_when_loading_to_sqlserver_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index 054236995a..e89226e2d3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -118,11 +118,11 @@ def test_given_managed_source_when_cloning_sqlserver_then_source_precedes_depend "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_customers(ctx):\n" - " return []\n" + " '''Test loader raw_customers.'''\n return []\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -132,7 +132,7 @@ def test_given_managed_source_when_cloning_sqlserver_then_source_precedes_depend " type: VARCHAR\n" ), "models/stg_customers.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_customers.', materialized view);\n\n" 'SELECT customer_id, first_name FROM __source("raw_customers")\n' ), }, @@ -294,7 +294,7 @@ def test_given_sqlserver_targets_when_running_diff_then_all_modes_execute_end_to f'schema = "{dev_schema}"\n' ), "models/fact_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model fact_orders.',\n" " materialized table,\n" " unique_key [order_id],\n" " cursor order_id,\n" @@ -436,14 +436,14 @@ def test_given_python_result_when_running_check_on_sqlserver_then_persists_node_ "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -559,7 +559,7 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta SqlServerScenarioLocalReplayE2ETestCase( description="captures SQL Server fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " CAST(CAST(event_ts AS DATE) AS DATETIME2) AS event_day,\n" @@ -570,7 +570,7 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta "GROUP BY customer_id, CAST(CAST(event_ts AS DATE) AS DATETIME2)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, CAST('2026-01-01 08:15:00' AS DATETIME2)" @@ -601,12 +601,12 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta description="reports SQL Server local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -709,7 +709,7 @@ def test_given_source_loader_project_when_building_on_sqlserver_then_model_reads ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -722,10 +722,10 @@ def test_given_source_loader_project_when_building_on_sqlserver_then_model_reads "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -781,7 +781,7 @@ def test_given_source_deferral_env_when_building_on_sqlserver_then_reads_prod_an ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -794,10 +794,10 @@ def test_given_source_deferral_env_when_building_on_sqlserver_then_reads_prod_an "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -1031,7 +1031,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1046,10 +1046,10 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1109,7 +1109,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1118,7 +1118,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1144,7 +1144,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1155,7 +1155,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1176,7 +1176,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1187,7 +1187,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1213,7 +1213,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_sqlserver_the repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 777eb95cc7..e980efa666 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -14,7 +14,7 @@ def build_dynamic_pivot_test_project_files() -> dict[str, str]: 'contract = "enforced"\n' ), "models/stg_order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_amounts.',\n" " database analytics,\n" " schema staging,\n" " columns (\n" @@ -30,7 +30,7 @@ def build_dynamic_pivot_test_project_files() -> dict[str, str]: "CAST('US' AS VARCHAR) AS country\n" ), "models/customer_category_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " database analytics,\n" " schema mart,\n" " columns (\n" @@ -86,7 +86,7 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s f"sql_analysis = {sql_analysis_value}\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "SELECT\n" " id,\n" " @mocked_amount() AS amount,\n" @@ -96,7 +96,7 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s 'FROM __source("raw")' ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "SELECT\n" " id,\n" " amount + 1 AS adjusted,\n" @@ -115,7 +115,8 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s "def real_status() -> str:\n" " return \"'active'\"\n" ), - "sources/raw.yml": "sources:\n - name: raw\n schema: main\n table: raw\n", + "sources/raw.yml": "sources:\n - name: raw\n description: Test source raw.\n" + " schema: main\n table: raw\n", "tests/unit/test_chain.sql": ( "TEST();\n\n" "WITH\n" @@ -146,10 +147,17 @@ def build_parameterized_test_project_files() -> dict[str, str]: "[connection]\n" 'database = "parameter_case_demo.duckdb"\n' ), - "models/orders.sql": 'MODEL ();\n\nSELECT status FROM __source("raw_orders")\n', - "models/customers.sql": ('MODEL ();\n\nSELECT status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT status FROM __source("raw_orders")\n' + ), + "models/customers.sql": ( + 'MODEL (description "Test model customers.");\n\n' + 'SELECT status FROM __source("raw_orders")\n' + ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( 'TEST (name "order_status", parameters (status string), cases (' @@ -181,14 +189,14 @@ def build_table_function_fixture_project_files() -> dict[str, str]: 'database = "table_function_fixture_demo.duckdb"\n' ), "functions/sql/customer_orders.sql": ( - "FUNCTION (\n" + "FUNCTION (description 'Test function customer_orders.',\n" " arguments (customer_id INTEGER),\n" " returns table (order_id INTEGER)\n" ");\n\n" "SELECT customer_id * 100 AS order_id\n" ), "models/orders.sql": ( - "MODEL (columns (order_id (type INTEGER)));\n\n" + "MODEL (description 'Test model orders.', columns (order_id (type INTEGER)));\n\n" 'SELECT order_id FROM __table_fn("customer_orders")(7)\n' ), "tests/unit/test_orders.sql": ( @@ -212,9 +220,16 @@ def build_mock_boundary_test_project_files() -> dict[str, str]: "[connection]\n" 'database = "mock_boundary_demo.duckdb"\n' ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", - "models/int_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n'), - "models/fact_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("int_orders")\n'), + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT 1 AS order_id\n", + "models/int_orders.sql": ( + 'MODEL (description "Test model int_orders.");\n\n' + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/fact_orders.sql": ( + 'MODEL (description "Test model fact_orders.");\n\n' + 'SELECT order_id FROM __ref("int_orders")\n' + ), "tests/unit/test_fact_orders.sql": ( "TEST();\n\n" "WITH\n" @@ -243,8 +258,9 @@ def build_unsatisfied_leaf_test_project_files() -> dict[str, str]: files: dict[str, str] = build_chain_test_project_files(sql_analysis_enabled=True) files["sources/raw.yml"] = ( "sources:\n" - " - name: raw\n schema: main\n table: raw\n" - " - name: unused\n schema: main\n table: unused\n" + " - name: raw\n description: Test source raw.\n schema: main\n table: raw\n" + " - name: unused\n description: Test source unused.\n" + " schema: main\n table: unused\n" ) files["tests/unit/test_chain.sql"] = ( "TEST();\n\n" @@ -270,11 +286,12 @@ def build_missing_mock_columns_project_files() -> dict[str, str]: 'database = "missing_fixture_columns.duckdb"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT customer_id, status FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT customer_id, status FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -301,10 +318,13 @@ def build_partial_source_fixture_project_files() -> dict[str, str]: "[connection]\n" 'database = "partial_source_fixture.duckdb"\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __source("raw_orders")\n' + ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -345,9 +365,13 @@ def build_partial_ref_fixture_project_files() -> dict[str, str]: 'database = "partial_ref_fixture.duckdb"\n' ), "models/stg_orders.sql": ( - "MODEL ();\n\nSELECT CAST(1 AS INTEGER) AS order_id, CAST(NULL AS VARCHAR) AS status\n" + "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(NULL AS VARCHAR) AS status\n" + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __ref("stg_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __ref("stg_orders")\n'), "tests/unit/test_orders.sql": ( "TEST();\n\n" "WITH\n" @@ -371,16 +395,20 @@ def build_open_schema_ref_fixture_project_files() -> dict[str, str]: 'database = "open_schema_ref_fixture.duckdb"\n' ), "models/stg_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_orders.',\n" " columns (\n" " order_id (type INTEGER),\n" " ),\n" ");\n\n" 'SELECT * FROM __source("raw_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __ref("stg_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __ref("stg_orders")\n' + ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( "TEST();\n\n" @@ -403,7 +431,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: 'database = "recursive_ref_fixture.duckdb"\n' ), "models/stg_order_links.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_links.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -413,7 +441,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: 'SELECT order_id, parent_order_id FROM __source("raw_order_links")\n' ), "models/order_roots.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_roots.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -436,7 +464,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: ), "sources/raw_order_links.yml": ( "sources:\n" - " - name: raw_order_links\n" + " - name: raw_order_links\n description: Test source raw_order_links.\n" " schema: main\n" " table: raw_order_links\n" " columns:\n" @@ -468,12 +496,13 @@ def build_partial_seed_fixture_project_files() -> dict[str, str]: 'database = "partial_seed_fixture.duckdb"\n' ), "models/countries.sql": ( - 'MODEL ();\n\nSELECT country_code, country_name FROM __seed("country_codes")\n' + 'MODEL (description "Test model countries.");\n\n' + 'SELECT country_code, country_name FROM __seed("country_codes")\n' ), "seeds/country_codes.csv": "country_code,country_name\nUS,United States\n", "seeds/schema.yml": ( "seeds:\n" - " - name: country_codes\n" + " - name: country_codes\n description: Test seed country_codes.\n" " columns:\n" " - name: country_code\n type: VARCHAR\n nullable: false\n" " - name: country_name\n type: VARCHAR\n nullable: true\n" @@ -494,7 +523,9 @@ def build_irrelevant_omitted_column_project_files() -> dict[str, str]: """Build a fixture omitting a known nullable column outside the compiled closure.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' + ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" "WITH\n" @@ -510,7 +541,7 @@ def build_cte_partial_source_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "WITH staged AS (\n" ' SELECT order_id, status FROM __source("raw_orders")\n' ")\n" @@ -524,7 +555,8 @@ def build_clause_partial_source_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders") WHERE status IS NULL\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id FROM __source("raw_orders") WHERE status IS NULL\n' ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" @@ -543,7 +575,9 @@ def build_star_partial_fixture_project_files() -> dict[str, str]: files["sources/raw_orders.yml"] = files["sources/raw_orders.yml"].replace( " columns:\n", " contract: enforced\n columns:\n" ) - files["models/orders.sql"] = 'MODEL ();\n\nSELECT * FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __source("raw_orders")\n' + ) return files @@ -592,7 +626,7 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: 'database = "untyped_null_fixture.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (order_year (type BIGINT)),\n" ");\n\n" @@ -601,7 +635,7 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -625,7 +659,7 @@ def build_contract_empty_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -668,7 +702,9 @@ def build_open_expected_empty_fixture_project_files() -> dict[str, str]: """Build an empty expected fixture without an authoritative model shape.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT * FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __source("raw_orders")\n' + ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" "WITH\n" @@ -691,16 +727,17 @@ def build_qualified_star_other_relation_project_files() -> dict[str, str]: "[connection]\n" 'database = "qualified_star_other_relation.duckdb"\n' ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS stg_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT 1 AS stg_id\n", "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "SELECT stg.*, raw.status\n" 'FROM __ref("stg_orders") stg\n' 'JOIN __source("raw_orders") raw ON TRUE\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -723,10 +760,13 @@ def build_mixed_case_partial_fixture_project_files() -> dict[str, str]: """Build a completed fixture with an as-written mixed-case unquoted alias.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT status, note FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT status, note FROM __source("raw_orders")\n' + ) files["sources/raw_orders.yml"] = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -753,7 +793,7 @@ def build_invalid_partial_fixture_project_files( files: dict[str, str] = build_partial_source_fixture_project_files() files["sources/raw_orders.yml"] = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -788,12 +828,12 @@ def build_incompatible_fixture_type_project_files(*, adapter_name: str) -> dict[ 'schema = "main"\n' ), "models/orders.sql": ( - 'MODEL (database "fixture_db", schema "main");\n\n' + 'MODEL (description "Test model orders.", database "fixture_db", schema "main");\n\n' 'SELECT item_ids FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -821,12 +861,12 @@ def build_complex_values_fixture_project_files(*, adapter_name: str) -> dict[str 'schema = "main"\n' ), "models/orders.sql": ( - 'MODEL (database "fixture_db", schema "main");\n\n' + 'MODEL (description "Test model orders.", database "fixture_db", schema "main");\n\n' 'SELECT order_id, mapping_text FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -880,11 +920,12 @@ def build_union_distinct_expected_project_files() -> dict[str, str]: 'database = "union_distinct_expected.duckdb"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT DISTINCT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT DISTINCT order_id FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -917,11 +958,12 @@ def build_transformed_collection_project_files() -> dict[str, str]: 'database = "transformed_collection.duckdb"\n' ), "models/order_statuses.sql": ( - 'MODEL ();\n\nSELECT list(status) AS statuses FROM __source("raw_orders")\n' + 'MODEL (description "Test model order_statuses.");\n\n' + 'SELECT list(status) AS statuses FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -948,7 +990,7 @@ def build_cte_derived_output_fixture_project_files() -> dict[str, str]: 'database = "cte_derived_output_fixture.duckdb"\n' ), "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "WITH raw_orders AS (\n" ' SELECT * FROM __source("raw_orders")\n' "), derived AS (\n" @@ -963,7 +1005,8 @@ def build_cte_derived_output_fixture_project_files() -> dict[str, str]: "SELECT order_id, normalized_status, loaded_at FROM final\n" ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( "TEST();\n\n" @@ -1013,10 +1056,11 @@ def build_assertion_test_project_files(*, failing: bool) -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT\n" " id AS order_id,\n" " amount\n" @@ -1050,7 +1094,8 @@ def build_macro_test_project_files() -> dict[str, str]: 'def normalize_status(value: str) -> str:\n return f"LOWER(TRIM({value}))"\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" + "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" ), "tests/unit/test_normalize_status.sql": ( 'TEST (mode macro, name "normalizes_status");\n\n' @@ -1081,25 +1126,26 @@ def build_diamond_chain_test_project_files(*, sql_analysis_enabled: bool) -> dic 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT id AS order_id, customer_id, amount FROM __source("raw_orders")\n' ), "models/large_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model large_orders.', materialized table);\n\n" 'SELECT order_id, customer_id, amount FROM __ref("stg_orders") WHERE amount >= 10\n' ), "models/small_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model small_orders.', materialized table);\n\n" "WITH picked AS (\n" ' SELECT order_id, customer_id, amount FROM __ref("stg_orders") WHERE amount < 10\n' ")\n" "SELECT * FROM picked -- small orders only\n" ), "models/customer_order_mix.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_order_mix.', materialized table);\n\n" "SELECT customer_id,\n" ' (SELECT COUNT(*) FROM __ref("large_orders") AS l\n' " WHERE l.customer_id = c.customer_id) AS large_count,\n" @@ -1150,17 +1196,20 @@ def build_deep_shared_missing_mock_project_files(*, layers: int) -> dict[str, st ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_customers\n schema: main\n table: raw_customers\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_customers\n description: Test source raw_customers.\n" + " schema: main\n table: raw_customers\n" ), "models/orders_00_left.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders_00_left.', materialized table);\n\n" "SELECT o.order_id, o.amount\n" 'FROM __source("raw_orders") AS o\n' 'JOIN __source("raw_customers") AS c ON c.customer_id = o.customer_id\n' ), "models/orders_00_right.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, amount FROM __ref("orders_00_left")\n' + 'MODEL (description "Test model orders_00_right.", ' + 'materialized table);\n\nSELECT order_id, amount FROM __ref("orders_00_left")\n' ), } for layer in range(1, layers + 1): @@ -1168,7 +1217,7 @@ def build_deep_shared_missing_mock_project_files(*, layers: int) -> dict[str, st previous_right: str = f"orders_{layer - 1:02d}_right" for side in ("left", "right"): files[f"models/orders_{layer:02d}_{side}.sql"] = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT a.order_id, a.amount + b.amount AS amount\n" f'FROM __ref("{previous_left}") AS a\n' f'JOIN __ref("{previous_right}") AS b ON a.order_id = b.order_id\n' @@ -1201,10 +1250,11 @@ def build_expected_column_subset_project_files( 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT id AS order_id, customer_id, status, amount * 2 AS amount\n" 'FROM __source("raw_orders")\n' ), @@ -1290,14 +1340,15 @@ def build_helper_scope_project_files() -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT id AS order_id, amount FROM __source("raw_orders")\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" 'SELECT order_id, amount * 2 AS amount FROM __ref("stg_orders")\n' ), } @@ -1394,11 +1445,13 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_events\n schema: main\n table: raw_events\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" ), "models/daily_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model daily_orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor order_date,\n" @@ -1412,7 +1465,7 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()\n" ), "models/order_lookback.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_lookback.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor scan_start,\n" @@ -1428,7 +1481,7 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str "WHERE order_date >= CAST(__cursor_start() AS DATE) - INTERVAL 7 DAY\n" ), "models/event_totals.sql": ( - "MODEL (\n" + "MODEL (description 'Test model event_totals.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor event_id,\n" @@ -1441,10 +1494,11 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str " AND event_id - 1 < __cursor_end() + 1\n" ), "models/plain_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT amount FROM __source("raw_orders")\n' + 'MODEL (description "Test model plain_orders.", ' + 'materialized table);\n\nSELECT amount FROM __source("raw_orders")\n' ), "models/hourly_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model hourly_orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" @@ -1488,7 +1542,8 @@ def build_cte_scope_project_files(*, queries: tuple[str, ...], expected: str) -> 'name = "cte_scope"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/chain.sql": ( 'TEST (name "order_chain");\nWITH ' @@ -1497,5 +1552,5 @@ def build_cte_scope_project_files(*, queries: tuple[str, ...], expected: str) -> ), } for index, query in enumerate(queries): - files[f"models/orders_{index}.sql"] = f"MODEL ();\n{query}" + files[f"models/orders_{index}.sql"] = f"MODEL (description 'Test model.');\n{query}" return files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py index 994276ecfa..875899670b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py @@ -15,9 +15,9 @@ 'name = "escape_strings"\nadapter = "duckdb"\n\n' '[connection]\ndatabase = "escape_strings.duckdb"\n' ), - "models/stg_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id\n", "models/order_names.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_names.', materialized table);\n\n" "SELECT E'O\\'Brien' AS customer_name, order_id\n" 'FROM __ref("stg_orders")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py index 32378e33a7..469d09acdd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py @@ -27,7 +27,7 @@ ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -35,7 +35,7 @@ " - name: status\n type: VARCHAR\n" ), "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" 'SELECT order_id, @normalize_status("status") AS status FROM __source("raw_orders")\n' ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py index 6066ef65c6..d880521376 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py @@ -30,8 +30,8 @@ def test_given_combined_sql_over_256k_when_testing_on_duckdb_then_passes( 'name = "large_sql_test"\nadapter = "duckdb"\n\n' '[connection]\ndatabase = "orders.duckdb"\n' ), - "models/raw_orders.sql": "MODEL ();\nSELECT 'orders' AS details\n", - "models/orders.sql": 'MODEL ();\nSELECT details FROM __ref("raw_orders")\n', + "models/raw_orders.sql": "MODEL (description 'Test model raw_orders.');\nSELECT 'orders' AS details\n", + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT details FROM __ref("raw_orders")\n', "tests/unit/test_orders.sql": ( 'TEST (name "large_orders");\nWITH __ref__raw_orders AS (\n' f"SELECT '{payload}' AS details\n), __expected__orders AS (\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py index 7084916745..2d5d6e4dc9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py @@ -39,18 +39,18 @@ def test_given_upstream_function_when_compiling_and_testing_then_mocked_chain_ex '[connection]\ndatabase = "orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "functions/sql/identity_value.sql": ( - "FUNCTION (arguments (value INTEGER), returns INTEGER);\nvalue\n" + "FUNCTION (description 'Test function identity_value.', arguments (value INTEGER), returns INTEGER);\nvalue\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n" 'SELECT __udf("identity_value")(order_id) AS order_id ' 'FROM __source("raw_orders")\n' ), "models/orders.sql": ( - 'MODEL (materialized table);\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\nSELECT * FROM __ref("stg_orders")\n' ), "tests/unit/orders_case.sql": ( 'TEST (name "orders_case");\nWITH\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py index 9481008692..7b39be0dc4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py @@ -112,7 +112,7 @@ def add_dbt_profile_downstream_model(*, sqlbuild_project_dir: Path) -> None: models_dir: Path = sqlbuild_project_dir / "models" models_dir.mkdir(exist_ok=True) (models_dir / "downstream_orders.sql").write_text( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model downstream_orders.', materialized table);\n\n" 'SELECT order_id FROM __dbt_ref("analytics", "dbt_orders")\n', encoding="utf-8", ) @@ -523,7 +523,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "sqlbuild_project.toml": project_toml, "models/current_customers.sql": build_current_customers_model_sql(plan="basic"), "models/current_customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region_id],\n" " snapshot_strategy timestamp,\n" @@ -541,7 +541,8 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st 'FROM __ref("current_customers")\n' ), "models/historical_customer_extracts.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model historical_customer_extracts.', " + "materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at, " "CAST('2026-01-02 00:00:00' AS TIMESTAMP) AS observed_at\n" @@ -553,7 +554,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "CAST('2026-01-04 00:00:00' AS TIMESTAMP) AS observed_at\n" ), "models/historical_customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -565,7 +566,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st 'FROM __ref("historical_customer_extracts")\n' ), "models/historical_membership_daily.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model historical_membership_daily.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS observed_at\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status, " @@ -578,7 +579,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "CAST('2026-01-03 00:00:00' AS TIMESTAMP) AS observed_at\n" ), "models/historical_membership_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_membership_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -600,7 +601,7 @@ def build_current_customers_model_sql(*, plan: str, updated_at: str = "2026-01-0 """Build the mutable current-state model for real-warehouse snapshot tests.""" return ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 10 AS region_id, " f"'{plan}' AS plan, CAST('{updated_at}' AS TIMESTAMP) AS updated_at\n" ) @@ -615,7 +616,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/current_check_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_check_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -628,7 +629,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/current_delete_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_delete_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -642,7 +643,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/historical_timestamp_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_timestamp_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -656,7 +657,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - ), "models/historical_check_daily.sql": build_historical_check_daily_model_sql(changed=False), "models/historical_check_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_check_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -676,12 +677,12 @@ def build_current_check_customers_model_sql(*, changed: bool) -> str: return { False: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status\n" ), True: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'paused' AS status\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status\n" ), @@ -693,14 +694,14 @@ def build_current_delete_customers_model_sql(*, changed: bool) -> str: return { False: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at\n" "UNION ALL SELECT 2 AS customer_id, 'trial' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at\n" ), True: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'pro' AS plan, " "CAST('2026-01-03 00:00:00' AS TIMESTAMP) AS updated_at\n" ), @@ -711,7 +712,7 @@ def build_historical_timestamp_extracts_model_sql(*, changed: bool) -> str: """Build mutable historical timestamp source model for live apply tests.""" sql: str = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at, " "CAST('2026-01-02 00:00:00' AS TIMESTAMP) AS observed_at\n" @@ -734,7 +735,7 @@ def build_historical_check_daily_model_sql(*, changed: bool) -> str: """Build mutable historical check source model for live apply tests.""" sql: str = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS observed_at\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status, " diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index a7d343e087..f6eacfa60a 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -51,7 +51,7 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) (factories_dir / "generated_nodes.py").write_text( @@ -68,10 +68,12 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: "def generated_nodes():", " @task(depends_on=model('orders'), tags=['daily'], group='python')", " def prepare_orders(ctx):", + " '''Test task prepare_orders.'''", " return ctx.result(payload={'rows': 1})", "", " @loader(name='warehouse_export', depends_on=(prepare_orders,))", " def warehouse_export(ctx):", + " '''Test loader warehouse_export.'''", " return [{'order_id': 1}]", "", " @asset(", @@ -81,10 +83,12 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: " columns=[{'name': 'order_id', 'type': 'integer'}],", " )", " def orders_export(ctx):", + " '''Test asset orders_export.'''", " return ctx.result(materialized=True)", "", " @check(depends_on=orders_export, tags=['quality'], group='exports')", " def check_orders_export(ctx):", + " '''Test check check_orders_export.'''", " return True", "", " return [prepare_orders, warehouse_export, orders_export, check_orders_export]", diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index d6b60417fb..ee0fa7932e 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -103,7 +103,7 @@ def test_given_model_level_column_audits_when_loading_dagster_assets_then_checks 'database = "column_audit_checks.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " columns (\n" " order_id (),\n" " customer_id (),\n" @@ -543,14 +543,14 @@ def test_given_chained_source_loader_when_dagster_selects_source_then_reuses_int " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) setup_result: subprocess.CompletedProcess[str] = run_sqb( @@ -634,14 +634,14 @@ def test_given_chained_source_loader_when_dagster_selects_source_without_interme " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) command_log_path: Path = tmp_path / "sqb_command_log.txt" diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py index 58c00bed17..cbb700083b 100644 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py +++ b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py @@ -55,7 +55,7 @@ def test_given_postgres_source_when_loading_with_ingestr_then_duckdb_target_has_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_pg_orders\n" + " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" " ingestr:\n" f' source_uri: "{postgres_uri(postgres_config)}"\n' f" source_table: public.{source_table}\n" @@ -113,7 +113,7 @@ def test_given_duckdb_source_when_loading_with_ingestr_to_duckdb_then_target_has ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" " ingestr:\n" f' source_uri: "duckdb:///{source_duckdb_path}"\n' " source_table: orders\n" @@ -175,7 +175,7 @@ def test_given_ingestr_duckdb_source_when_building_with_load_then_model_reads_lo ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -186,7 +186,7 @@ def test_given_ingestr_duckdb_source_when_building_with_load_then_model_reads_lo " source_table: orders\n" ), "models/stg_duckdb_orders.sql": """ -MODEL (materialized table); +MODEL (description "Test model stg_duckdb_orders.", materialized table); SELECT order_id, status FROM __source("raw_duckdb_orders") """.strip() @@ -250,7 +250,7 @@ def test_given_ingestr_source_when_building_with_load_then_model_reads_loaded_so ), "sources/raw.yml": ( "sources:\n" - " - name: raw_pg_orders\n" + " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -261,7 +261,7 @@ def test_given_ingestr_source_when_building_with_load_then_model_reads_loaded_so f" source_table: public.{source_table}\n" ), "models/stg_pg_orders.sql": """ -MODEL (materialized table); +MODEL (description "Test model stg_pg_orders.", materialized table); SELECT order_id, status FROM __source("raw_pg_orders") """.strip() @@ -327,7 +327,7 @@ def test_given_duckdb_source_when_loading_with_ingestr_then_postgres_target_has_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" f" schema: {target_schema}\n" " ingestr:\n" f' source_uri: "duckdb:///{source_duckdb_path}"\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index bb4537f6df..100649dbf9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -24,7 +24,7 @@ REPLAY_FULL: str = " replay_on_change full,\n" RELATION: str = f"main.{MODEL_NAME}" _INCREMENTAL_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy {strategy},\n" " unique_key order_id,\n" @@ -36,7 +36,7 @@ ");\n\n" ) _SNAPSHOT_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [order_id],\n" " snapshot_strategy timestamp,\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py index d047d909fb..42315ffa80 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py @@ -137,7 +137,7 @@ def test_given_unsafe_declared_rename_when_building_then_build_stops_before_chan ColumnMigrationCompileErrorTestCase( description="table materialization", model_sql=( - "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + "MODEL (description 'Test model.', materialized table, columns (revenue (migrate_from amount)));\n" 'SELECT amount AS revenue FROM __source("raw_orders")\n' ), expected_fragment="only valid for incremental and snapshot models", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 1c7ef28b7b..900532fb1c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -38,7 +38,8 @@ def write_compile_startup_project(project_dir: Path) -> None: models_dir: Path = project_dir / "models" models_dir.mkdir() (models_dir / "orders.sql").write_text( - "MODEL (materialized table);\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\nSELECT 1 AS order_id\n", + encoding="utf-8", ) @@ -110,7 +111,7 @@ def prepare_contract_project(tmp_path: Path, *, prod_connection_toml: str = "") models.mkdir() sources.mkdir() _ = (models / "orders.sql").write_text( - """MODEL ( + """MODEL (description "Test model orders.", materialized table -- keep model metadata columns ( @@ -240,10 +241,13 @@ def write_retention_policy_project(*, project_dir: Path, target_lines: tuple[str encoding="utf-8", ) _ = (project_dir / "models" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + encoding="utf-8", ) _ = (project_dir / "models" / "order_view.sql").write_text( - 'MODEL (materialized view);\n\nSELECT order_id FROM __ref("orders")\n', encoding="utf-8" + 'MODEL (description "Test model order_view.", ' + 'materialized view);\n\nSELECT order_id FROM __ref("orders")\n', + encoding="utf-8", ) @@ -292,10 +296,10 @@ def write_dropped_relation_project( ) _ = (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n", encoding="utf-8", @@ -305,7 +309,7 @@ def write_dropped_relation_project( ) _ = (project_dir / "seeds" / "lookups.yml").write_text( "seeds:\n" - " - name: order_statuses\n" + " - name: order_statuses\n description: Test seed order_statuses.\n" " columns:\n" " - name: status_code\n" " type: INTEGER\n" @@ -394,7 +398,7 @@ def dropped_relation_microbatch_sql(*, batch_concurrency: int) -> str: """Render an integer-cursor microbatch model over the raw order source.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" @@ -419,7 +423,9 @@ def compile_duckdb_projection( (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text(f"MODEL ();\nSELECT {projection}") + (models / "orders.sql").write_text( + f"MODEL (description 'Test model orders.');\nSELECT {projection}" + ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) @@ -439,7 +445,7 @@ def write_expression_source_project( ) (project_dir / "sources" / "orders.yml").write_text( "sources:\n" - " - name: typed_orders\n" + " - name: typed_orders\n description: Test source typed_orders.\n" f' expression: "{source_expression}"\n' " columns:\n" " - name: order_id\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 72d48a5e89..b5aac79091 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -96,6 +96,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -104,7 +105,7 @@ _FORCE_LINES: dict[bool, str] = {True: " migrate_force true,\n", False: ""} _DAILY_TOTALS_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_date,\n" @@ -116,7 +117,7 @@ ");\n\n" ) _SNAPSHOT_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [order_id],\n" " snapshot_strategy timestamp,\n" @@ -151,7 +152,7 @@ def incremental_orders_sql( migrate_from, f' migrate_from "{migrate_from}",\n' ) + _FORCE_LINES.get(migrate_force, "") return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -529,7 +530,7 @@ def enriched_view_sql(*, upstream: str, alias: str, materialized: str = "view") """Return a view or table that selects every order column through a table alias.""" return ( - f"MODEL (materialized {materialized});\n\n" + f"MODEL (description 'Test model.', materialized {materialized});\n\n" f"SELECT {alias}.order_id, {alias}.order_date, {alias}.amount_cents " f'FROM __ref("{upstream}") AS {alias}\n' ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index f0eaac192f..14ac0ae2c9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -461,7 +461,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="force on a table", model_sql=( - 'MODEL (materialized table, migrate_from "stg_orders", migrate_force true);\n' + 'MODEL (description "Test model.", materialized table, migrate_from "stg_orders", migrate_force true);\n' "SELECT 1 AS id\n" ), expected_fragment="migrate_force is only valid for incremental and snapshot models", @@ -469,7 +469,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="force on a view", model_sql=( - 'MODEL (materialized view, migrate_from "stg_orders", migrate_force true);\n' + 'MODEL (description "Test model.", materialized view, migrate_from "stg_orders", migrate_force true);\n' "SELECT 1 AS id\n" ), expected_fragment="migrate_force is only valid for incremental and snapshot models", @@ -477,7 +477,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="column alias on a table that is not migrating", model_sql=( - "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + "MODEL (description 'Test model.', materialized table, columns (revenue (migrate_from amount)));\n" "SELECT 1 AS revenue\n" ), expected_fragment="or for table and view models that declare migrate_from themselves", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 63c7799123..5b95c8f51f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -34,7 +34,7 @@ ).lstrip() _FACT_ORDER: tuple[str, ...] = ("required", "origin_archived", "view_created", "view_dropped") _TABLE_SQL: str = ( - "MODEL (materialized table{extra});\n" + "MODEL (description 'Test model.', materialized table{extra});\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -206,7 +206,8 @@ def disabled_table_sql(*, migrate_from: str) -> str: """Return a table model that declares its rename with old-name views turned off.""" return ( - f"MODEL (materialized table, migrate_from {migrate_from}, old_name_view false);\n" + "MODEL (description 'Test model.', " + f"materialized table, migrate_from {migrate_from}, old_name_view false);\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -225,7 +226,7 @@ def claiming_view_models() -> dict[str, str]: return { DESTINATION_MODEL: table_sql(migrate_from=ORIGIN_MODEL), ORIGIN_MODEL: ( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT order_id, amount_cents * 2 AS doubled FROM __source("raw_orders")\n' ), } @@ -236,7 +237,7 @@ def aliased_orders_sql(*, alias: str, migrate_from: str | None = None) -> str: migration: str = {None: ""}.get(migrate_from, f", migrate_from {migrate_from}") return ( - f"MODEL (materialized table, alias {alias}{migration});\n" + f"MODEL (description 'Test model.', materialized table, alias {alias}{migration});\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -264,7 +265,7 @@ def aliased_table_sql(*, extra_column: str = "") -> str: """Return the renamed table model with ``amount`` renamed to ``revenue`` in its old view.""" return ( - f"MODEL (materialized table, migrate_from {ORIGIN_MODEL}, " + f"MODEL (description 'Test model.', materialized table, migrate_from {ORIGIN_MODEL}, " "columns (revenue (migrate_from amount_cents)));\n" f'SELECT order_id, amount_cents AS revenue{extra_column} FROM __source("raw_orders")\n' ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py index ffb7d41daf..902a1c9a70 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py @@ -14,7 +14,7 @@ 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "demo.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (\n" @@ -25,7 +25,7 @@ "SELECT 1 AS order_id, 5 AS amount\n" ), "models/order_totals.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, amount * 2 AS doubled FROM __ref("orders")\n' + 'MODEL (description "Test model order_totals.", materialized table);\n\nSELECT order_id, amount * 2 AS doubled FROM __ref("orders")\n' ), "tests/unit/test_order_totals.sql": ( "TEST ();\nWITH\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py index 63a25020f4..66a6e8826c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py @@ -49,12 +49,12 @@ def test_given_typed_dependency_when_compiling_cte_then_preserves_binding_and_ou sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" ' expression: "(SELECT CAST(1 AS BIGINT) AS order_id)"\n' " columns:\n - name: order_id\n type: BIGINT\n" ) (models / "selected_orders.sql").write_text( - "MODEL (columns (result (type BOOLEAN)));\n" + "MODEL (description 'Test model selected_orders.', columns (result (type BOOLEAN)));\n" 'WITH selected AS (SELECT order_id FROM __source("orders"))\n' f"SELECT {test_case.projection} AS result FROM selected\n" ) @@ -83,7 +83,7 @@ def test_given_multiple_typed_model_headers_when_compiling_then_cli_preserves_he models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "staged_orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model staged_orders.',\n" " materialized table,\n" " columns (order_id (type INTEGER, nullable false)),\n" ' pre_hooks [inline_sql("SELECT 1")],\n' @@ -92,7 +92,7 @@ def test_given_multiple_typed_model_headers_when_compiling_then_cli_preserves_he encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized view,\n" " tags [core, 'daily orders'],\n" " columns (order_id (type DECIMAL(10,2))),\n" @@ -141,7 +141,7 @@ def test_given_model_macro_when_compiling_with_sql_rules_then_expands_once_and_p " return 'NULL'\n" ) (tmp_path / "models" / "orders" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id WHERE 1 = @missing_value()\n" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id WHERE 1 = @missing_value()\n" ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -173,7 +173,7 @@ def test_given_unicode_model_when_compiling_twice_then_cli_preserves_unchanged_a model_path: Path = tmp_path / "models" / "products.sql" model_path.parent.mkdir() model_path.write_text( - "MODEL (materialized table);\nSELECT 'café' AS product_name \n\n", + "MODEL (description 'Test model products.', materialized table);\nSELECT 'café' AS product_name \n\n", encoding="utf-8", ) @@ -235,7 +235,7 @@ def test_given_supported_snowflake_aggregation_when_compiling_then_project_succe model: Path = tmp_path / "models" / "order_summary.sql" model.parent.mkdir() model.write_text( - f"MODEL (database warehouse, schema analytics);\n\n{test_case.query_sql}", + f"MODEL (description 'Test model order_summary.', database warehouse, schema analytics);\n\n{test_case.query_sql}", encoding="utf-8", ) @@ -279,7 +279,7 @@ def test_given_nullable_output_when_compiling_contract_then_only_schema_nullabil model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" f" columns ({test_case.column_sql}),\n" ");\n\n" @@ -319,7 +319,7 @@ def test_given_relative_project_directory_when_compiling_then_private_macro_is_v 'def order_id() -> str:\n return "CAST(1 AS INTEGER)"\n', encoding="utf-8" ) (project_dir / "models" / "orders" / "orders.sql").write_text( - "MODEL (materialized table);\nSELECT @order_id() AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\nSELECT @order_id() AS order_id\n", encoding="utf-8", ) monkeypatch.chdir(tmp_path) @@ -350,17 +350,17 @@ def test_given_sql_test_chain_when_compiling_then_native_planner_writes_complete models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "stg_orders.sql").write_text( - 'MODEL (materialized table);\nSELECT order_id, amount FROM __source("raw_orders")\n', + 'MODEL (description "Test model stg_orders.", materialized table);\nSELECT order_id, amount FROM __source("raw_orders")\n', encoding="utf-8", ) (models_dir / "orders.sql").write_text( - 'MODEL (materialized table);\nSELECT order_id, amount FROM __ref("stg_orders")\n', + 'MODEL (description "Test model orders.", materialized table);\nSELECT order_id, amount FROM __ref("stg_orders")\n', encoding="utf-8", ) sources_dir: Path = tmp_path / "sources" sources_dir.mkdir() (sources_dir / "raw_orders.yml").write_text( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n", + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n", encoding="utf-8", ) tests_dir: Path = tmp_path / "tests" / "unit" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py index 5ce7a0bf1e..b2d4e72943 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py @@ -68,7 +68,7 @@ def test_given_derived_native_facts_when_compiling_then_contract_and_execution_a models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type BIGINT, nullable false)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (order_id (type BIGINT, nullable false)));\n" + test_case.query_sql ) @@ -115,9 +115,15 @@ def test_given_star_over_inferred_upstream_when_compiling_then_downstream_bindin (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') models: Path = tmp_path / "models" models.mkdir() - (models / "staged_orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id") - (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") - (models / "order_readers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + (models / "staged_orders.sql").write_text( + "MODEL (description 'Test model staged_orders.');\nSELECT 1 AS order_id" + ) + (models / "wide_orders.sql").write_text( + f"MODEL (description 'Test model wide_orders.');\n{test_case.star_model_sql}" + ) + (models / "order_readers.sql").write_text( + f"MODEL (description 'Test model order_readers.');\n{test_case.downstream_sql}" + ) for _ in ("cold", "warm"): exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -154,16 +160,18 @@ def test_given_star_over_derived_table_when_compiling_then_lineage_matches_proje seeds.mkdir() (seeds / "order_labels.csv").write_text("order_id,label\n1,first\n") (seeds / "order_labels.yml").write_text( - "seeds:\n - name: order_labels\n columns:\n" + "seeds:\n - name: order_labels\n description: Test seed order_labels.\n columns:\n" " - name: order_id\n type: INTEGER\n" " - name: label\n type: VARCHAR\n" ) models: Path = tmp_path / "models" models.mkdir() (models / "staged_orders.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount" + "MODEL (description 'Test model staged_orders.');\nSELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount" + ) + (models / "wide_orders.sql").write_text( + f"MODEL (description 'Test model wide_orders.');\n{test_case.star_model_sql}" ) - (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") for _ in ("cold", "warm"): exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -226,11 +234,13 @@ def test_given_snapshot_reader_when_compiling_then_validity_columns_are_known( models: Path = tmp_path / "models" models.mkdir() (models / "customer_snapshot.sql").write_text( - "MODEL (materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, " + "MODEL (description 'Test model customer_snapshot.', materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, " f"updated_at updated_at{test_case.snapshot_config});\n" "SELECT 1 AS customer_id, 'pro' AS plan, TIMESTAMP '2026-01-01' AS updated_at" ) - (models / "current_customers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + (models / "current_customers.sql").write_text( + f"MODEL (description 'Test model current_customers.');\n{test_case.downstream_sql}" + ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -334,14 +344,14 @@ def test_given_input_and_output_share_name_when_compiling_then_lineage_keeps_inp sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: INTEGER\n" " - name: amount\n type: DOUBLE\n" " - name: status\n type: VARCHAR\n" ) (models / "order_totals.sql").write_text( - "MODEL (contract enforced, columns (order_id (type INTEGER), " + "MODEL (description 'Test model order_totals.', contract enforced, columns (order_id (type INTEGER), " "status (type VARCHAR), total (type DOUBLE)));\n" f"{test_case.query_prefix}" "SELECT order_id, " @@ -438,7 +448,7 @@ def test_given_bound_wildcard_or_using_join_when_compiling_then_facts_match_exec sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: BIGINT\n nullable: false\n" " - name: category\n type: VARCHAR\n" @@ -447,7 +457,8 @@ def test_given_bound_wildcard_or_using_join_when_compiling_then_facts_match_exec models: Path = tmp_path / "models" models.mkdir() (models / "selected_orders.sql").write_text( - f"MODEL (contract enforced, columns ({test_case.output_contract}));\n" + test_case.query_sql + f"MODEL (description 'Test model selected_orders.', contract enforced, columns ({test_case.output_contract}));\n" + + test_case.query_sql ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -490,7 +501,7 @@ def test_given_object_wildcard_when_compiling_then_lineage_preserves_payload_inp sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: BIGINT\n" " - name: amount\n type: DOUBLE\n" @@ -498,7 +509,7 @@ def test_given_object_wildcard_when_compiling_then_lineage_preserves_payload_inp models: Path = tmp_path / "models" models.mkdir() (models / "selected_orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type BIGINT, nullable true)));\n" + "MODEL (description 'Test model selected_orders.', contract enforced, columns (order_id (type BIGINT, nullable true)));\n" + test_case.query_sql ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index f3c0e4990d..6b230cdad2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -39,9 +39,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu config_path.write_text(config, encoding="utf-8") models: Path = tmp_path / "models" models.mkdir() - header: str = ( - "MODEL (materialized table, contract enforced, columns (order_id (type INTEGER)));\n" - ) + header: str = "MODEL (description 'Test model.', materialized table, contract enforced, columns (order_id (type INTEGER)));\n" for index in range(test_case.model_count): (models / f"orders_{index:03}.sql").write_text( header + "SELECT CAST(1 AS INTEGER) AS order_id\n", encoding="utf-8" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py index 5e8ef31528..59d14755df 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py @@ -167,7 +167,7 @@ def test_given_parameterized_physical_type_when_generating_then_writes_valid_mod database: Path = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table);\n" + "MODEL (description 'Test model orders.', materialized table);\n" "SELECT CAST(1 AS INTEGER) AS id, CAST(1 AS DECIMAL(10,2)) AS amount\n", encoding="utf-8", ) @@ -289,7 +289,7 @@ def test_given_conflicting_contract_when_overwriting_then_physical_shape_replace _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - """MODEL ( + """MODEL (description "Test model orders.", materialized table columns ( id (type BIGINT, description "identifier") @@ -603,7 +603,7 @@ def test_given_cli_vars_when_writing_then_validation_recompile_uses_same_values( _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table);\n" + "MODEL (description 'Test model orders.', materialized table);\n" "SELECT CAST(@@identifier AS INTEGER) AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) @@ -647,7 +647,7 @@ def test_given_unresolved_additive_type_conflict_when_writing_then_file_is_not_r _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table, columns (id (type BIGINT), name (type VARCHAR)));\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (type BIGINT), name (type VARCHAR)));\n" "SELECT CAST(1 AS INTEGER) AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) @@ -698,7 +698,7 @@ def test_given_relative_project_directory_when_diffing_contract_then_private_mac model_path: Path = project_dir / "models" / "orders" / "orders.sql" model_path.parent.mkdir() _ = model_path.write_text( - "MODEL (materialized table);\nSELECT @order_id() AS id, CAST('a' AS VARCHAR) AS name\n", + "MODEL (description 'Test model orders.', materialized table);\nSELECT @order_id() AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) macro_path: Path = model_path.parent / "_sqlbuild" / "_macros" / "order_id.py" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 62ce0ec451..13e71a9007 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -53,7 +53,7 @@ def test_given_project_when_running_command_then_discovery_skips_eager_column_lo project_dir=tmp_path, source_expression="(SELECT 1 AS order_id, 'open' AS status)", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py index 4502280b6d..236231f93d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py @@ -28,7 +28,7 @@ ) _INCREMENTAL_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key id,\n" @@ -39,7 +39,7 @@ ");\n\n" 'SELECT id, ordered_at FROM __source("raw_orders")\n' ) -_TABLE_SQL: str = 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' +_TABLE_SQL: str = 'MODEL (description "Test model.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' _MICROBATCH_CONCURRENCY_SETTINGS: str = ( "\n[settings]\nconcurrency = 3\nmicrobatch_concurrency = true\n" ) @@ -69,7 +69,7 @@ ), DroppedRelationRecoveryTestCase( description="view", - model_sql='MODEL (materialized view);\n\nSELECT id FROM __source("raw_orders")\n', + model_sql='MODEL (description "Test model.", materialized view);\n\nSELECT id FROM __source("raw_orders")\n', drop_sql="DROP VIEW main.orders", expected_action="create_view", expected_rows=((1,), (2,), (3,)), @@ -77,7 +77,7 @@ DroppedRelationRecoveryTestCase( description="snapshot", model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [id],\n" " snapshot_strategy timestamp,\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index af96e4f2dd..10c641c8ad 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -35,7 +35,9 @@ write_snowflake_format_test, ) -_UNFORMATTED_ORDERS_SQL: str = "MODEL (materialized table);\nselect 1 as order_id\n" +_UNFORMATTED_ORDERS_SQL: str = ( + 'MODEL (description "Test model.", materialized table);\nselect 1 as order_id\n' +) _LAYOUT_ONLY_BODY: str = 'SELECT ALL\n order_id id,\n amount,\nFROM __source("raw_orders") o;\n' @@ -271,6 +273,7 @@ def test_given_table_function_argument_when_formatting_then_intrinsic_restores_a function.parent.mkdir(parents=True) function.write_text( """FUNCTION ( + description "Test function expand_order.", arguments (order_id INTEGER), returns table (order_id INTEGER) ); @@ -811,7 +814,8 @@ def test_given_format_selectors_when_scoping_then_paths_and_default_exclusions_a models: Path = tmp_path / "models" models.mkdir() (models / "customers.sql").write_text( - "MODEL (materialized table, columns (customer_id (type INTEGER),));\n" + 'MODEL (description "Customers.", materialized table, ' + "columns (customer_id (type INTEGER),));\n" "SELECT 1 AS customer_id\n", encoding="utf-8", ) @@ -1004,7 +1008,7 @@ def test_given_file_paths_when_formatting_then_only_named_files_change( LeadingCteCommentFormatIntegrationTestCase( description="comment block before a middle CTE stays above it and compiles", authored_sql=( - "MODEL (materialized view);\n\n" + 'MODEL (description "Test model.", materialized view);\n\n' 'WITH\nupstream AS (\n SELECT *\n FROM __ref("raw_orders")\n),\n\n' "-- Explains the next CTE: line one,\n-- line two.\n" "distinct_rows AS (\n SELECT order_id, customer_id\n FROM upstream\n" @@ -1027,7 +1031,7 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe models: Path = tmp_path / "models" models.mkdir() (models / "raw_orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id, 2 AS customer_id\n", + 'MODEL (description "Test model raw_orders.", materialized view);\n\nSELECT 1 AS order_id, 2 AS customer_id\n', encoding="utf-8", ) model_path: Path = models / "distinct_orders.sql" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 6554f61b17..4cc8af7813 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -83,6 +83,7 @@ @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 1, "status": "loaded"}] """ @@ -91,6 +92,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -105,10 +107,11 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 7, "status": "loaded"}] """, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), } @@ -225,6 +228,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. columns: - name: order_id type: INTEGER @@ -238,7 +242,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_UNSELECTED_MANAGED_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_MISSING_PROJECT_FILES, "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' ), } @@ -270,6 +274,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -290,10 +295,12 @@ def raw_orders(ctx): ], ) def fetch_orders(ctx): + '''Test loader fetch_orders.''' return [{"order_id": 7, "status": "intermediate"}] @loader(depends_on=[fetch_orders]) def raw_orders(ctx): + '''Test loader raw_orders.''' rows = ctx.query( f"SELECT order_id, status FROM {ctx.loader(fetch_orders).destination}" ).fetchall() @@ -319,6 +326,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: external_raw table: raw_orders managed: true @@ -362,6 +370,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table expression: | @@ -382,7 +391,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_SQL_FUNCTION_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_PROJECT_FILES, "functions/sql/order_statuses.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_statuses.", returns table ( order_id INTEGER, status VARCHAR @@ -422,6 +431,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table audits: [source_status] @@ -443,7 +453,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_BUILD_WITH_FUNCTION_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_PROJECT_FILES, "functions/sql/order_statuses.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_statuses.", returns table ( order_id INTEGER, status VARCHAR @@ -475,7 +485,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_SCENARIO_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_MISSING_PROJECT_FILES, "tests/scenarios/source_mock_pass.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario source_mock_pass."); WITH __source__raw_orders AS ( @@ -492,7 +502,7 @@ def raw_orders(ctx): _BUILD_RUN_AUTO_LOAD_SELECTION_PROJECT_FILES: dict[str, str] = { **_BUILD_RUN_AUTO_LOAD_PROJECT_FILES, "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' ), } @@ -503,6 +513,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' raise RuntimeError("loader exploded") """, } @@ -514,6 +525,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = "reload" if ctx.is_reload else "incremental" return [{"order_id": 7, "status": status}] """, @@ -524,6 +536,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -532,6 +545,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -546,10 +560,11 @@ def raw_orders(ctx): @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [{"customer_id": 10, "name": "Ada"}] """, "models/stg_customers.sql": ( - 'MODEL (materialized table);\n\nSELECT customer_id, name FROM __source("raw_customers")' + 'MODEL (description "Test model stg_customers.", materialized table);\n\nSELECT customer_id, name FROM __source("raw_customers")' ), } @@ -558,6 +573,7 @@ def raw_customers(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true columns: - name: order_id @@ -571,13 +587,14 @@ def raw_customers(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.execute_sql( "CREATE OR REPLACE TABLE raw_orders AS " "SELECT 7 AS order_id, 'self-managed' AS status" ) """, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), } @@ -591,6 +608,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -605,6 +623,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -641,6 +660,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -649,6 +669,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -658,6 +679,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -665,6 +687,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -699,6 +722,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -713,6 +737,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -1859,6 +1884,7 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1873,6 +1899,7 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -1909,6 +1936,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1917,6 +1945,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -1926,6 +1955,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -1933,6 +1963,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -1967,6 +1998,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1981,6 +2013,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -2071,6 +2104,7 @@ def test_given_source_loader_when_running_load_then_writes_source_table( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2085,6 +2119,7 @@ def test_given_source_loader_when_running_load_then_writes_source_table( @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -2121,6 +2156,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2129,6 +2165,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -2138,6 +2175,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -2145,6 +2183,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -2179,6 +2218,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2193,6 +2233,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -2320,6 +2361,7 @@ def test_given_source_loader_when_running_pipeline_then_drops_stale_staging_firs "sources/raw.yml": """ sources: - name: raw_append_events + description: Test source raw_append_events. managed: true write_strategy: append cursor_column: event_id @@ -2335,6 +2377,7 @@ def test_given_source_loader_when_running_pipeline_then_drops_stale_staging_firs @loader def raw_append_events(ctx): + '''Test loader raw_append_events.''' if ctx.current_cursor_value is None: return [{"event_id": 1, "cursor_seen": "none"}] next_id = int(ctx.current_cursor_value) + 1 @@ -2351,6 +2394,7 @@ def raw_append_events(ctx): "sources/raw.yml": """ sources: - name: raw_merge_customers + description: Test source raw_merge_customers. managed: true write_strategy: merge unique_key: customer_id @@ -2369,6 +2413,7 @@ def raw_append_events(ctx): @loader def raw_merge_customers(ctx): + '''Test loader raw_merge_customers.''' if ctx.current_cursor_value is None: return [ {"customer_id": 1, "name": "old", "updated_at": 1}, @@ -2392,6 +2437,7 @@ def raw_merge_customers(ctx): "sources/raw.yml": """ sources: - name: raw_merge_composite + description: Test source raw_merge_composite. managed: true write_strategy: merge unique_key: [entity_id, source] @@ -2412,6 +2458,7 @@ def raw_merge_customers(ctx): @loader def raw_merge_composite(ctx): + '''Test loader raw_merge_composite.''' if ctx.current_cursor_value is None: return [{"entity_id": 1, "source": "api", "value": "old", "version": 1}] return [ @@ -2432,6 +2479,7 @@ def raw_merge_composite(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_events + description: Test source raw_delete_insert_events. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2447,6 +2495,7 @@ def raw_merge_composite(ctx): @loader def raw_delete_insert_events(ctx): + '''Test loader raw_delete_insert_events.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "old-outside-low"}, @@ -2477,6 +2526,7 @@ def raw_delete_insert_events(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_ts + description: Test source raw_delete_insert_ts. managed: true write_strategy: delete_insert cursor_column: event_at @@ -2494,6 +2544,7 @@ def raw_delete_insert_events(ctx): @loader def raw_delete_insert_ts(ctx): + '''Test loader raw_delete_insert_ts.''' if ctx.current_cursor_value is None: return [ {"event_at": datetime(2026, 1, 1, 0, 0, 0), "status": "outside-low"}, @@ -2525,6 +2576,7 @@ def raw_delete_insert_ts(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_empty + description: Test source raw_delete_insert_empty. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2540,6 +2592,7 @@ def raw_delete_insert_ts(ctx): @loader def raw_delete_insert_empty(ctx): + '''Test loader raw_delete_insert_empty.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "existing"}, @@ -2561,6 +2614,7 @@ def raw_delete_insert_empty(ctx): "sources/raw.yml": """ sources: - name: raw_no_cursor + description: Test source raw_no_cursor. managed: true write_strategy: append columns: @@ -2577,6 +2631,7 @@ def raw_delete_insert_empty(ctx): @loader def raw_no_cursor(ctx): + '''Test loader raw_no_cursor.''' global run_count run_count += 1 return [{"run_number": run_count, "cursor_seen": str(ctx.current_cursor_value)}] @@ -2592,6 +2647,7 @@ def raw_no_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_empty_cursor + description: Test source raw_empty_cursor. managed: true write_strategy: append cursor_column: event_id @@ -2607,6 +2663,7 @@ def raw_no_cursor(ctx): @loader def raw_empty_cursor(ctx): + '''Test loader raw_empty_cursor.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -2622,6 +2679,7 @@ def raw_empty_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_missing_cursor_target + description: Test source raw_missing_cursor_target. managed: true write_strategy: append cursor_column: event_id @@ -2637,6 +2695,7 @@ def raw_empty_cursor(ctx): @loader def raw_missing_cursor_target(ctx): + '''Test loader raw_missing_cursor_target.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -2653,6 +2712,7 @@ def raw_missing_cursor_target(ctx): "sources/raw.yml": """ sources: - name: raw_reload_context + description: Test source raw_reload_context. managed: true write_strategy: table columns: @@ -2665,6 +2725,7 @@ def raw_missing_cursor_target(ctx): @loader def raw_reload_context(ctx): + '''Test loader raw_reload_context.''' return [{"is_reload": ctx.is_reload}] """, } @@ -2674,6 +2735,7 @@ def raw_reload_context(ctx): "sources/raw.yml": """ sources: - name: raw_cursor_overrides + description: Test source raw_cursor_overrides. managed: true write_strategy: table columns: @@ -2692,6 +2754,7 @@ def raw_reload_context(ctx): @loader def raw_cursor_overrides(ctx): + '''Test loader raw_cursor_overrides.''' return [{ "start_ts": None if ctx.start_cursor_ts is None else ctx.start_cursor_ts.isoformat(), "end_ts": None if ctx.end_cursor_ts is None else ctx.end_cursor_ts.isoformat(), @@ -2712,6 +2775,7 @@ def raw_cursor_overrides(ctx): "sources/raw.yml": """ sources: - name: raw_append_events + description: Test source raw_append_events. managed: true write_strategy: append cursor_column: event_id @@ -2727,6 +2791,7 @@ def raw_cursor_overrides(ctx): @loader def raw_append_events(ctx): + '''Test loader raw_append_events.''' if ctx.current_cursor_value is None: return [{"event_id": 1, "cursor_seen": "none"}] next_id = int(ctx.current_cursor_value) + 1 @@ -2743,6 +2808,7 @@ def raw_append_events(ctx): "sources/raw.yml": """ sources: - name: raw_merge_customers + description: Test source raw_merge_customers. managed: true write_strategy: merge unique_key: customer_id @@ -2761,6 +2827,7 @@ def raw_append_events(ctx): @loader def raw_merge_customers(ctx): + '''Test loader raw_merge_customers.''' if ctx.current_cursor_value is None: return [ {"customer_id": 1, "name": "old", "updated_at": 1}, @@ -2784,6 +2851,7 @@ def raw_merge_customers(ctx): "sources/raw.yml": """ sources: - name: raw_merge_composite + description: Test source raw_merge_composite. managed: true write_strategy: merge unique_key: [entity_id, source] @@ -2804,6 +2872,7 @@ def raw_merge_customers(ctx): @loader def raw_merge_composite(ctx): + '''Test loader raw_merge_composite.''' if ctx.current_cursor_value is None: return [{"entity_id": 1, "source": "api", "value": "old", "version": 1}] return [ @@ -2824,6 +2893,7 @@ def raw_merge_composite(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_events + description: Test source raw_delete_insert_events. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2839,6 +2909,7 @@ def raw_merge_composite(ctx): @loader def raw_delete_insert_events(ctx): + '''Test loader raw_delete_insert_events.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "old-outside-low"}, @@ -2869,6 +2940,7 @@ def raw_delete_insert_events(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_ts + description: Test source raw_delete_insert_ts. managed: true write_strategy: delete_insert cursor_column: event_at @@ -2886,6 +2958,7 @@ def raw_delete_insert_events(ctx): @loader def raw_delete_insert_ts(ctx): + '''Test loader raw_delete_insert_ts.''' if ctx.current_cursor_value is None: return [ {"event_at": datetime(2026, 1, 1, 0, 0, 0), "status": "outside-low"}, @@ -2917,6 +2990,7 @@ def raw_delete_insert_ts(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_empty + description: Test source raw_delete_insert_empty. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2932,6 +3006,7 @@ def raw_delete_insert_ts(ctx): @loader def raw_delete_insert_empty(ctx): + '''Test loader raw_delete_insert_empty.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "existing"}, @@ -3220,6 +3295,7 @@ def delete_insert_cursor_spy( "sources/raw.yml": """ sources: - name: raw_no_cursor + description: Test source raw_no_cursor. managed: true write_strategy: append columns: @@ -3236,6 +3312,7 @@ def delete_insert_cursor_spy( @loader def raw_no_cursor(ctx): + '''Test loader raw_no_cursor.''' global run_count run_count += 1 return [{"run_number": run_count, "cursor_seen": str(ctx.current_cursor_value)}] @@ -3251,6 +3328,7 @@ def raw_no_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_empty_cursor + description: Test source raw_empty_cursor. managed: true write_strategy: append cursor_column: event_id @@ -3266,6 +3344,7 @@ def raw_no_cursor(ctx): @loader def raw_empty_cursor(ctx): + '''Test loader raw_empty_cursor.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -3281,6 +3360,7 @@ def raw_empty_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_missing_cursor_target + description: Test source raw_missing_cursor_target. managed: true write_strategy: append cursor_column: event_id @@ -3296,6 +3376,7 @@ def raw_empty_cursor(ctx): @loader def raw_missing_cursor_target(ctx): + '''Test loader raw_missing_cursor_target.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -3498,6 +3579,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context "sources/raw.yml": """ sources: - name: raw_a + description: Test source raw_a. managed: true write_strategy: table columns: @@ -3506,6 +3588,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context - name: connection_id type: BIGINT - name: raw_b + description: Test source raw_b. managed: true write_strategy: table columns: @@ -3514,6 +3597,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context - name: connection_id type: BIGINT - name: raw_c + description: Test source raw_c. managed: true write_strategy: table columns: @@ -3533,17 +3617,20 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context @loader def raw_a(ctx): + '''Test loader raw_a.''' barrier.wait(timeout=1) time.sleep(0.05) return [{"source_name": "raw_a", "connection_id": id(ctx.connection)}] @loader def raw_b(ctx): + '''Test loader raw_b.''' barrier.wait(timeout=1) return [{"source_name": "raw_b", "connection_id": id(ctx.connection)}] @loader def raw_c(ctx): + '''Test loader raw_c.''' return [{"source_name": "raw_c", "connection_id": id(ctx.connection)}] """, }, @@ -3617,6 +3704,7 @@ def test_given_multiple_source_loaders_when_running_pipeline_then_uses_concurren "sources/raw.yml": """ sources: - name: raw_join + description: Test source raw_join. managed: true write_strategy: table columns: @@ -3636,6 +3724,7 @@ def test_given_multiple_source_loaders_when_running_pipeline_then_uses_concurren @loader(write_strategy='table', columns=[{'name': 'id', 'type': 'INTEGER'}]) def fetch_a(ctx): + '''Test loader fetch_a.''' barrier.wait(timeout=1) time.sleep(0.05) with lock: @@ -3644,6 +3733,7 @@ def fetch_a(ctx): @loader(write_strategy='table', columns=[{'name': 'id', 'type': 'INTEGER'}]) def fetch_b(ctx): + '''Test loader fetch_b.''' barrier.wait(timeout=1) with lock: finished.add('b') @@ -3651,6 +3741,7 @@ def fetch_b(ctx): @loader(depends_on=[fetch_a, fetch_b]) def raw_join(ctx): + '''Test loader raw_join.''' return [{'upstream_count': len(finished)}] """, }, @@ -3738,6 +3829,7 @@ def test_given_loader_dag_when_running_pipeline_then_independent_branches_overla "sources/raw.yml": """ sources: - name: raw_inferred + description: Test source raw_inferred. managed: true write_strategy: table columns: @@ -3754,6 +3846,7 @@ def test_given_loader_dag_when_running_pipeline_then_independent_branches_overla @loader def raw_inferred(ctx): + '''Test loader raw_inferred.''' yield { "id": 1, "flag": True, @@ -3832,6 +3925,7 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri "sources/raw.yml": """ sources: - name: raw_multi_yield + description: Test source raw_multi_yield. managed: true write_strategy: table columns: @@ -3846,6 +3940,7 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri @loader def raw_multi_yield(ctx): + '''Test loader raw_multi_yield.''' yield {"id": 1, "status": "first"} yield {"id": 2, "status": "second"} """, @@ -3885,6 +3980,7 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ "sources/raw.yml": """ sources: - name: raw_batched_yield + description: Test source raw_batched_yield. managed: true write_strategy: table load_batch_size: 1 @@ -3900,6 +3996,7 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ @loader def raw_batched_yield(ctx): + '''Test loader raw_batched_yield.''' yield {"id": 1, "status": "first"} yield {"id": 2, "status": "second", "late_flag": True} yield {"id": 3, "status": "third", "late_flag": False} @@ -3979,6 +4076,7 @@ def test_given_generator_loader_uses_batch_size_when_running_pipeline_then_appen "sources/raw.yml": """ sources: - name: raw_missing_known + description: Test source raw_missing_known. managed: true write_strategy: table load_batch_size: 1 @@ -3994,6 +4092,7 @@ def test_given_generator_loader_uses_batch_size_when_running_pipeline_then_appen @loader def raw_missing_known(ctx): + '''Test loader raw_missing_known.''' yield {"id": 1, "status": "first"} yield {"id": 2} """, @@ -4011,6 +4110,7 @@ def raw_missing_known(ctx): "sources/raw.yml": """ sources: - name: raw_empty_generator + description: Test source raw_empty_generator. managed: true write_strategy: table load_batch_size: 1 @@ -4026,6 +4126,7 @@ def raw_missing_known(ctx): @loader def raw_empty_generator(ctx): + '''Test loader raw_empty_generator.''' if False: yield {"id": 1, "status": "unreachable"} """, @@ -4043,6 +4144,7 @@ def raw_empty_generator(ctx): "sources/raw.yml": """ sources: - name: raw_late_null + description: Test source raw_late_null. managed: true write_strategy: table load_batch_size: 1 @@ -4056,6 +4158,7 @@ def raw_empty_generator(ctx): @loader def raw_late_null(ctx): + '''Test loader raw_late_null.''' yield {"id": 1, "late_note": None} yield {"id": 2, "late_note": "filled"} """, @@ -4073,6 +4176,7 @@ def raw_late_null(ctx): "sources/raw.yml": """ sources: - name: raw_batch_size_two + description: Test source raw_batch_size_two. managed: true write_strategy: table load_batch_size: 2 @@ -4086,6 +4190,7 @@ def raw_late_null(ctx): @loader def raw_batch_size_two(ctx): + '''Test loader raw_batch_size_two.''' yield {"id": 1} yield {"id": 2} yield {"id": 3} @@ -4105,6 +4210,7 @@ def raw_batch_size_two(ctx): "sources/raw.yml": """ sources: - name: raw_default_batch + description: Test source raw_default_batch. managed: true write_strategy: table columns: @@ -4117,6 +4223,7 @@ def raw_batch_size_two(ctx): @loader def raw_default_batch(ctx): + '''Test loader raw_default_batch.''' yield {"id": 1} yield {"id": 2} yield {"id": 3} @@ -4194,6 +4301,7 @@ def test_given_batched_loader_variants_when_running_pipeline_then_writes_expecte "sources/raw.yml": """ sources: - name: raw_many_rows + description: Test source raw_many_rows. managed: true write_strategy: table columns: @@ -4206,6 +4314,7 @@ def test_given_batched_loader_variants_when_running_pipeline_then_writes_expecte @loader def raw_many_rows(ctx): + '''Test loader raw_many_rows.''' for value in range(1001): yield {"id": value} """, @@ -4255,6 +4364,7 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ "sources/raw.yml": """ sources: - name: raw_empty + description: Test source raw_empty. managed: true write_strategy: table columns: @@ -4269,6 +4379,7 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ @loader def raw_empty(ctx): + '''Test loader raw_empty.''' return [] """, }, @@ -4314,6 +4425,7 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec "sources/raw.yml": """ sources: - name: raw_self_managed + description: Test source raw_self_managed. managed: true columns: - name: id @@ -4325,6 +4437,7 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec @loader def raw_self_managed(ctx): + '''Test loader raw_self_managed.''' ctx.execute_sql("CREATE OR REPLACE TABLE raw_self_managed AS SELECT 1 AS id") """, }, @@ -4366,6 +4479,7 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta "sources/raw.yml": """ sources: - name: raw_conflict + description: Test source raw_conflict. managed: true write_strategy: table """.strip() @@ -4375,6 +4489,7 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta @loader def raw_conflict(ctx): + '''Test loader raw_conflict.''' return [ {"id": 1}, {"id": "two"}, @@ -4391,6 +4506,7 @@ def raw_conflict(ctx): "sources/raw.yml": """ sources: - name: raw_missing_strategy + description: Test source raw_missing_strategy. managed: true """.strip() + "\n", @@ -4399,6 +4515,7 @@ def raw_conflict(ctx): @loader def raw_missing_strategy(ctx): + '''Test loader raw_missing_strategy.''' return [{"id": 1}] """, }, @@ -4412,6 +4529,7 @@ def raw_missing_strategy(ctx): "sources/raw.yml": """ sources: - name: raw_unexpected_none + description: Test source raw_unexpected_none. managed: true write_strategy: table """.strip() @@ -4421,6 +4539,7 @@ def raw_missing_strategy(ctx): @loader def raw_unexpected_none(ctx): + '''Test loader raw_unexpected_none.''' return None """, }, @@ -4457,6 +4576,7 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle "sources/raw.yml": """ sources: - name: raw_batched_conflict + description: Test source raw_batched_conflict. managed: true write_strategy: table load_batch_size: 1 @@ -4467,6 +4587,7 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle @loader def raw_batched_conflict(ctx): + '''Test loader raw_batched_conflict.''' yield {"id": 1} yield {"id": "two"} """, @@ -4487,6 +4608,7 @@ def raw_batched_conflict(ctx): "sources/raw.yml": """ sources: - name: raw_batched_non_dict + description: Test source raw_batched_non_dict. managed: true write_strategy: table load_batch_size: 1 @@ -4497,6 +4619,7 @@ def raw_batched_conflict(ctx): @loader def raw_batched_non_dict(ctx): + '''Test loader raw_batched_non_dict.''' yield {"id": 1} yield ("id", 2) """, @@ -4512,6 +4635,7 @@ def raw_batched_non_dict(ctx): "sources/raw.yml": """ sources: - name: raw_append_failure + description: Test source raw_append_failure. managed: true write_strategy: append load_batch_size: 1 @@ -4525,6 +4649,7 @@ def raw_batched_non_dict(ctx): @loader def raw_append_failure(ctx): + '''Test loader raw_append_failure.''' yield {"id": 1} yield {"id": "two"} """, @@ -4545,6 +4670,7 @@ def raw_append_failure(ctx): "sources/raw.yml": """ sources: - name: raw_merge_failure + description: Test source raw_merge_failure. managed: true write_strategy: merge unique_key: id @@ -4561,6 +4687,7 @@ def raw_append_failure(ctx): @loader def raw_merge_failure(ctx): + '''Test loader raw_merge_failure.''' yield {"id": 1, "status": "new"} yield {"id": "two", "status": "bad"} """, @@ -4581,6 +4708,7 @@ def raw_merge_failure(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_failure + description: Test source raw_delete_insert_failure. managed: true write_strategy: delete_insert cursor_column: id @@ -4597,6 +4725,7 @@ def raw_merge_failure(ctx): @loader def raw_delete_insert_failure(ctx): + '''Test loader raw_delete_insert_failure.''' yield {"id": 1, "status": "new"} yield {"id": "two", "status": "bad"} """, @@ -4671,6 +4800,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -4690,6 +4820,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4707,6 +4838,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -4726,9 +4858,11 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4773,6 +4907,7 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err "sources/raw.yml": """ sources: - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4794,6 +4929,7 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 33fb11fea3..80909ee47d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -117,14 +117,16 @@ def test_given_exact_internal_edge_exception_when_compiling_then_live_edge_passe models: Path = tmp_path / "models" models.mkdir() (models / "commerce__stg__orders.sql").write_text( - 'MODEL ();\nSELECT order_id FROM __ref("commerce__int_enriched__orders")\n', + 'MODEL (description "Test model commerce__stg__orders.");\nSELECT order_id FROM __ref("commerce__int_enriched__orders")\n', encoding="utf-8", ) (models / "commerce__int_enriched__orders.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model commerce__int_enriched__orders.');\nSELECT 1 AS order_id\n", + encoding="utf-8", ) (models / "commerce__mart__summary.sql").write_text( - "MODEL ();\nSELECT 1 AS order_count\n", encoding="utf-8" + "MODEL (description 'Test model commerce__mart__summary.');\nSELECT 1 AS order_count\n", + encoding="utf-8", ) live_exit: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) @@ -257,7 +259,7 @@ def test_given_constant_backed_decision_and_numeric_alias_when_compiling_then_ru model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - """MODEL ( + """MODEL (description "Test model orders.", constants (_large_batch 7), ); @@ -446,7 +448,7 @@ def test_given_enforced_contract_when_compiling_explicit_output_rule_then_enforc model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" f" columns ({test_case.columns_sql}),\n" ");\n\n" @@ -492,7 +494,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "stg_order_amounts.sql").write_text( - "MODEL (database analytics, schema analytics, columns (customer_id (type INTEGER), " + "MODEL (description 'Test model stg_order_amounts.', database analytics, schema analytics, columns (customer_id (type INTEGER), " "category (type VARCHAR), " 'amount (type "DECIMAL(12,2)")));\n' "SELECT CAST(1 AS INTEGER) AS customer_id, " @@ -502,7 +504,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ ) dynamic_model: Path = models_dir / "customer_category_amounts.sql" dynamic_model.write_text( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " database analytics,\n" " schema analytics,\n" " columns (customer_id (type INTEGER)),\n" @@ -582,7 +584,7 @@ def test_given_contract_rules_when_compiling_then_contract_is_enforced_typed_and model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" f" columns ({test_case.columns_sql}),\n" ");\n\n" "SELECT CAST(1 AS INTEGER) AS order_id, CAST('ready' AS VARCHAR) AS status\n", @@ -1190,7 +1192,7 @@ def test_given_sql_function_argument_when_running_qualification_rule_then_argume function: Path = tmp_path / "functions" / "sql" / "orders_by_date.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function orders_by_date.", arguments (start_date DATE), returns table (order_id INTEGER) ); @@ -1236,7 +1238,7 @@ def test_given_table_function_argument_when_running_alias_rule_then_left_alias_i function: Path = tmp_path / "functions" / "sql" / "expand_order.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function expand_order.", arguments (order_id INTEGER), returns table (order_id INTEGER) ); @@ -1525,7 +1527,7 @@ def test_given_seed_and_function_references_when_running_source_rule_then_they_a function: Path = tmp_path / "functions" / "sql" / "normalize_quantity.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function normalize_quantity.", arguments (quantity INTEGER), returns INTEGER, ); @@ -2210,7 +2212,9 @@ def test_given_custom_rule_module_constant_edit_when_compiling_then_cached_resul ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_source: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule @@ -2259,8 +2263,12 @@ def test_given_focused_project_rule_result_when_compiling_full_project_then_subs ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") - (models / "customers.sql").write_text("MODEL ();\nSELECT 1 AS customer_id\n", encoding="utf-8") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + (models / "customers.sql").write_text( + "MODEL (description 'Test model customers.');\nSELECT 1 AS customer_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_file.write_text( @@ -2302,9 +2310,11 @@ def test_given_non_model_sql_violation_when_compiling_then_compile_is_authoritat ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) (tmp_path / "models" / "customers.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", encoding="utf-8" ) audit: Path = tmp_path / "audits" / "singular" / "order_sample.sql" audit.parent.mkdir(parents=True) @@ -2339,9 +2349,12 @@ def test_given_exception_for_other_model_when_compiling_selection_then_exception ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) (models / "customers.sql").write_text( - "MODEL ();\nSELECT customer_id FROM customers LIMIT 1\n", encoding="utf-8" + "MODEL (description 'Test model customers.');\nSELECT customer_id FROM customers LIMIT 1\n", + encoding="utf-8", ) exit_code: int = main( @@ -2372,6 +2385,7 @@ def test_given_typed_unselected_model_when_planning_selection_then_rules_use_sel (sources / "customers.yml").write_text( """sources: - name: raw_customers + description: Test source raw_customers. schema: raw table: customers columns: @@ -2383,12 +2397,12 @@ def test_given_typed_unselected_model_when_planning_selection_then_rules_use_sel models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type INTEGER)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (order_id (type INTEGER)));\n" "SELECT CAST(1 AS INTEGER) AS order_id\n", encoding="utf-8", ) (models / "customers.sql").write_text( - """MODEL ( + """MODEL (description "Test model customers.", contract enforced, columns (customer_id (type VARCHAR)), ); @@ -2485,7 +2499,10 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai ) model: Path = project_dir / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT order_id FROM orders LIMIT 1\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT order_id FROM orders LIMIT 1\n", + encoding="utf-8", + ) monkeypatch.chdir(tmp_path) first_exit: int = main(["--project-dir", "project", "compile", "--json"]) @@ -2515,7 +2532,7 @@ def test_given_unused_inline_suppression_when_compiling_then_stale_directive_is_ model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL ();\n-- sqb: ignore SQBRSQL004 because this fixture checks stale directives\n" + "MODEL (description 'Test model orders.');\n-- sqb: ignore SQBRSQL004 because this fixture checks stale directives\n" "SELECT 1 AS order_id\n", encoding="utf-8", ) @@ -2544,7 +2561,9 @@ def test_given_unselected_custom_rule_options_when_compiling_then_configuration_ ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_file.write_text( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py index 0344d39ee5..4d42869376 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py @@ -35,7 +35,9 @@ def test_given_sql_and_custom_exceptions_when_compiling_cold_and_warm_then_both_ ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id LIMIT 1\n") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id LIMIT 1\n" + ) rules: Path = tmp_path / "rules" rules.mkdir() (rules / "architecture.py").write_text( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py b/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py index 52690a0815..c3da610ca9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py @@ -22,7 +22,7 @@ description="names already known at compile time add no rebinding", source_expression="(SELECT 1 AS order_id, 'open' AS status)", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), expected_exit_code=0, @@ -32,7 +32,7 @@ description="names hidden from compile rebind and reject a missing column", source_expression="(SELECT * FROM main.raw_orders)", model_sql=( - 'MODEL (materialized table);\nSELECT o.missing FROM __source("typed_orders") AS o\n' + 'MODEL (description "Test model.", materialized table);\nSELECT o.missing FROM __source("typed_orders") AS o\n' ), expected_exit_code=1, expected_rebinding=True, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py b/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py index c9a273d56a..ed904eb422 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py @@ -23,13 +23,13 @@ [ TargetRetentionViewsTestCase( description="view skips inherited target retention", - view_header="MODEL (materialized view);", + view_header="MODEL (description 'Test model.', materialized view);", expected_exit_code=0, expected_fragment="Project compiled", ), TargetRetentionViewsTestCase( description="explicit view retention is still rejected", - view_header="MODEL (materialized view, time_travel_retention 7d);", + view_header="MODEL (description 'Test model.', materialized view, time_travel_retention 7d);", expected_exit_code=1, expected_fragment="managed time_travel_retention is not valid for views", ), @@ -44,7 +44,8 @@ def test_given_target_retention_default_when_compiling_views_then_only_explicit_ (tmp_path / "models").mkdir() (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") (tmp_path / "models" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + encoding="utf-8", ) (tmp_path / "models" / "order_view.sql").write_text( f'{test_case.view_header}\n\nSELECT order_id FROM __ref("orders")\n', encoding="utf-8" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 77dce05d9b..b5dd9c4a9a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -61,7 +61,7 @@ class SemanticCompileCase: upstream: str downstream: str expected_code: str | None - header: str = "MODEL (materialized view);\n" + header: str = "MODEL (description 'Test model.', materialized view);\n" expected_diagnostic_count: int = 1 expected_column: int | None = None diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index debd19f9d3..906227cb77 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -156,9 +156,12 @@ def compile_reshaped_star_model( models: Path = project_dir / "models" models.mkdir() for name, sql in RESHAPED_STAR_UPSTREAM_MODELS.items(): - (models / f"{name}.sql").write_text(f"MODEL (materialized table);\n{sql}", encoding="utf-8") + (models / f"{name}.sql").write_text( + f"MODEL (description 'Test model.', materialized table);\n{sql}", encoding="utf-8" + ) (models / "reshaped_orders.sql").write_text( - f"MODEL (materialized table);\n{query_sql}", encoding="utf-8" + f"MODEL (description 'Test model reshaped_orders.', materialized table);\n{query_sql}", + encoding="utf-8", ) project: CompiledProject = compile_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), @@ -176,7 +179,9 @@ def lineage_source_pairs(column: CompiledLineageColumnFact) -> frozenset[tuple[s ) -_SHARED_BINDING_UPSTREAM_SQL: str = "MODEL (materialized view); SELECT 1 AS id, 2.5 AS amount" +_SHARED_BINDING_UPSTREAM_SQL: str = ( + "MODEL (description 'Test model.', materialized view); SELECT 1 AS id, 2.5 AS amount" +) def write_shared_binding_project(*, project_dir: Path, test_case: SharedBindingQueryCase) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py index e671bc3cd8..0a3f70c70f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py @@ -25,7 +25,7 @@ "sqlbuild_project.toml": ( 'name = "gate_demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n' ), - "models/valid_codes.sql": "MODEL (materialized table);\nSELECT 'A' AS code\n", + "models/valid_codes.sql": "MODEL (description 'Test model valid_codes.', materialized table);\nSELECT 'A' AS code\n", "audits/singular/reconcile.sql": ( 'AUDIT ();\nSELECT o.* FROM __ref("orders") o JOIN __ref("valid_codes") v USING (code)\n' ), @@ -37,13 +37,13 @@ [ SingularAuditAttachmentIntegrationTestCase( description="unrelated models without a gate edge run the singular audit at the end", - orders_header="MODEL (materialized table);", + orders_header="MODEL (description 'Test model.', materialized table);", generic_audit_files={}, expected_attachment=("end", None), ), SingularAuditAttachmentIntegrationTestCase( description="an attached audit's gate edge does not move the singular audit", - orders_header="MODEL (materialized table, audits [order_check]);", + orders_header="MODEL (description 'Test model.', materialized table, audits [order_check]);", generic_audit_files=_ORDER_CHECK, expected_attachment=("end", None), ), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py index 15d13ced77..103fd56042 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py @@ -46,7 +46,7 @@ def order_quality(): ] """, "models/orders.sql": """ -MODEL (audit_factories [order_quality]); +MODEL (description "Test model orders.", audit_factories [order_quality]); SELECT 10 AS amount """, }, @@ -57,7 +57,7 @@ def order_quality(): tmp_path, { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [ expression_is_true (name "positive_amount", expression "amount > 0", severity error), expression_is_true (name "bounded_amount", expression "amount < 1000", severity warn), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py index ce365a20bb..c4ff500ce8 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py @@ -28,12 +28,16 @@ def test_given_inferred_producer_when_binding_consumers_then_avoids_deferred_req ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS quantity\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS id, 2 AS quantity\n" ) (tmp_path / "models").mkdir() producer: Path = tmp_path / "models/orders.sql" - producer.write_text('MODEL (materialized view); SELECT id FROM __source("raw_orders")') - (tmp_path / "models/report.sql").write_text("MODEL (materialized view); " + test_case.sql) + producer.write_text( + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' + ) + (tmp_path / "models/report.sql").write_text( + "MODEL (description 'Test model report.', materialized view); " + test_case.sql + ) original: Callable[..., Any] = project.get_schema_validations requests: list[Any] = [] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py index 592026618f..3305168812 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py @@ -33,18 +33,20 @@ def test_given_completed_shape_when_an_unrelated_model_changes_then_cached_colum ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n table: orders\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n - name: quantity\n type: INTEGER\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n - name: quantity\n type: INTEGER\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/order_inputs.sql").write_text( - 'MODEL (materialized view); SELECT id, quantity FROM __source("raw_orders")' + 'MODEL (description "Test model order_inputs.", materialized view); SELECT id, quantity FROM __source("raw_orders")' ) (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view);\n" + "MODEL (description 'Test model orders.', materialized view);\n" + test_case.sql.replace('__source("raw_orders")', '__ref("order_inputs")') ) edited: Path = tmp_path / "models/customers.sql" - edited.write_text("MODEL (materialized view); SELECT 1 AS id") + edited.write_text( + "MODEL (description 'Test model customers.', materialized view); SELECT 1 AS id" + ) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] assert main(args) == 0 cold: dict[str, Any] = json.loads(capsys.readouterr().out) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index d88e58c44b..0719d710ad 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -24,19 +24,19 @@ '[targets.dev]\nschema = "dev_schema"\n' ), "models/staging/stg_orders.sql": ( - "MODEL (materialized table\ntags [staging]);\n\nSELECT 1 AS order_id" + "MODEL (description 'Test model stg_orders.', materialized table\ntags [staging]);\n\nSELECT 1 AS order_id" ), "models/staging/stg_customers.sql": ( - "MODEL (materialized table\ntags [staging]);\n\nSELECT 1 AS customer_id" + "MODEL (description 'Test model stg_customers.', materialized table\ntags [staging]);\n\nSELECT 1 AS customer_id" ), "models/intermediate/int_orders.sql": ( - 'MODEL (materialized table\ntags [core]);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model int_orders.", materialized table\ntags [core]);\n\nSELECT order_id FROM __ref("stg_orders")' ), "models/marts/fact_orders.sql": ( - 'MODEL (materialized table\ntags [core, mart]);\n\nSELECT order_id FROM __ref("int_orders")' + 'MODEL (description "Test model fact_orders.", materialized table\ntags [core, mart]);\n\nSELECT order_id FROM __ref("int_orders")' ), "models/marts/dim_customers.sql": ( - 'MODEL (materialized table\ntags [mart]);\n\nSELECT customer_id FROM __ref("stg_customers")' + 'MODEL (description "Test model dim_customers.", materialized table\ntags [mart]);\n\nSELECT customer_id FROM __ref("stg_customers")' ), } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index f72f6980c9..306d2278f3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -31,12 +31,12 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' ) (tmp_path / "sources").mkdir() source: Path = tmp_path / "sources/orders.yml" source.write_text( - f"sources:\n - name: raw_orders\n expression: {test_case.expression}\n - name: raw_customers\n expression: {test_case.expression}\n" + f"sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.expression}\n - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.expression}\n" ) original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch calls: list[object] = [] @@ -81,13 +81,13 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( "sources:\n" - f" - name: raw_orders\n expression: {test_case.orders_expression}\n" - f" - name: raw_customers\n expression: {test_case.customers_expression}\n" + f" - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.orders_expression}\n" + f" - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.customers_expression}\n" ) original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch batches: list[tuple[str, ...]] = [] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 1d74739134..bf14bcf478 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -64,7 +64,7 @@ def test_given_active_adapter_when_macro_renders_constant_then_uses_adapter_sql( "def region_array(ctx) -> str:\n return ctx.render_constant('regions')\n" ), "models/summary.sql": ( - "MODEL (materialized view, database warehouse, schema analytics);\n\n" + "MODEL (description 'Test model summary.', materialized view, database warehouse, schema analytics);\n\n" "SELECT @region_array() AS regions" ), }, @@ -328,7 +328,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return str(ctx.constants['minimum_quantity'])\n" ), "models/summary.sql": ( - "MODEL (materialized view, audits [minimum_quantity]);\n\n" + "MODEL (description 'Test model summary.', materialized view, audits [minimum_quantity]);\n\n" "SELECT @minimum_quantity() AS minimum_quantity" ), "tests/unit/summary.sql": ( @@ -367,7 +367,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return str(ctx.constants['minimum_quantity'])\n" ), "models/products/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @minimum_quantity() AS minimum_quantity" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @minimum_quantity() AS minimum_quantity" ), }, expected_error_fragment="Constant 'minimum_quantity'.*is inaccessible", @@ -424,7 +424,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return ctx.render_constant('maximum_quantity')\n" ), "models/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @missing_quantity() AS maximum_quantity" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @missing_quantity() AS maximum_quantity" ), }, expected_error_fragment=( @@ -445,7 +445,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " )\n" ), "models/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @missing_status() AS status" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @missing_status() AS status" ), }, expected_error_fragment=( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index eb77e2825d..45ab004033 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -44,7 +44,7 @@ "sqlbuild_project.toml": _PROJECT_TOML, "models/_macros/counting.py": _COUNTING_MACRO_MODULE, "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id, @order_tag() AS tag" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id, @order_tag() AS tag" ), }, expected_macro_import_count=1, @@ -91,7 +91,7 @@ def test_given_side_effect_macro_when_compiling_manifest_then_imports_macros_onc "def _helper() -> str:\n return shared()\n\n" "def order_column() -> str:\n return _helper()\n" ), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT @order_column()", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT @order_column()", }, macro_name="order_column", expected_dependencies=("shared",), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 2f6bdd4759..f7dee2a733 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -53,7 +53,7 @@ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT '12' AS amount\n" " columns:\n" " - name: amount\n" @@ -68,6 +68,7 @@ ), "models/orders.sql": ( "MODEL (\n" + ' description "Test model orders.",\n' " contract enforced,\n" " model_schema order_amount,\n" ");\n\n" @@ -153,7 +154,9 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i description="single table model with no schema defaults to adapter schema", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id" + ), }, expected_models={ "orders": ExpectedModelEntry( @@ -172,7 +175,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/active_orders.sql": ( - "MODEL (materialized view);\n\nSELECT order_id FROM orders WHERE status = 'active'" + "MODEL (description 'Test model active_orders.', materialized view);\n\nSELECT order_id FROM orders WHERE status = 'active'" ), }, expected_models={ @@ -192,7 +195,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": ( - "MODEL (\n materialized table\n schema analytics\n" + "MODEL (description 'Test model orders.',\n materialized table\n schema analytics\n" " database warehouse\n);\n\n" "SELECT 1 AS order_id" ), @@ -213,9 +216,11 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i description="two models with ref dependency resolves ref to qualified name", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/stg_orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/stg_orders.sql": ( + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id" + ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, expected_models={ @@ -242,10 +247,10 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _FILE_PROJECT_TOML, "sources/raw.yml": ( - "sources:\n - name: raw_payments\n schema: main\n table: payments\n" + "sources:\n - name: raw_payments\n description: Test source raw_payments.\n schema: main\n table: payments\n" ), "models/stg_payments.sql": ( - 'MODEL (materialized table);\n\nSELECT payment_id FROM __source("raw_payments")' + 'MODEL (description "Test model stg_payments.", materialized table);\n\nSELECT payment_id FROM __source("raw_payments")' ), }, expected_models={ @@ -265,10 +270,10 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/staging/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS order_id" + "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id" ), "models/marts/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, expected_models={ @@ -329,7 +334,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i " )\n" ), "models/policy_summary.sql": ( - "MODEL (materialized view);\n\nSELECT @policy_columns()" + "MODEL (description 'Test model policy_summary.', materialized view);\n\nSELECT @policy_columns()" ), }, expected_models={ @@ -423,12 +428,12 @@ def test_given_project_files_when_running_compile_pipeline_then_produces_valid_o description="run selector resolves Python task without SQL-only planning", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result(payload={'ok': True})\n" + " '''Test task prepare_orders.'''\n return ctx.result(payload={'ok': True})\n" ), }, expected_models={}, @@ -465,7 +470,7 @@ def test_given_python_run_selector_when_running_compile_pipeline_then_tracks_pyt description="reports compile progress from compile pipeline", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, expected_progress_prefixes=("Compiling project...", "Compiled project."), ) @@ -498,7 +503,7 @@ def test_given_progress_callback_when_running_compile_pipeline_then_reports_comp description="reports compile progress from project graph build", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, expected_progress_prefixes=("Compiling project...", "Compiled project."), ) @@ -543,7 +548,7 @@ def test_given_progress_callback_when_building_project_graph_then_reports_compil 'database = "TEST_DB"\n' 'schema = "TEST_SCHEMA"\n' ), - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, expected_error_fragment="snowflake execution requires explicit target database", ) @@ -585,7 +590,7 @@ def test_given_snowflake_local_override_without_target_namespace_when_compiling_ 'database = "${ENV:TEST_DB}"\n' 'schema = "${ENV:TEST_SCHEMA}"\n' ), - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, expected_database="LOCAL_DB", expected_schema="LOCAL_SCHEMA", @@ -633,9 +638,11 @@ def test_given_snowflake_local_target_namespace_when_compiling_then_targets_reso "[targets.prod]\n" 'schema = "prod_schema"\n' ), - "models/stg_orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/stg_orders.sql": ( + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id" + ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, defer_to="prod", @@ -696,15 +703,15 @@ def test_given_project_with_defer_to_when_compiling_then_resolves_refs_to_deferr "sql_analysis = true\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, amount FROM __source("raw")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT id, amount FROM __source("raw")' ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "WITH local_helper AS (SELECT 1 AS one) " 'SELECT id, amount + one AS adjusted FROM __ref("stg_orders") ' "CROSS JOIN local_helper" ), - "sources/raw.yml": "sources:\n - name: raw\n schema: main\n table: raw\n", + "sources/raw.yml": "sources:\n - name: raw\n description: Test source raw.\n schema: main\n table: raw\n", "tests/unit/test_chain.sql": ( "TEST();\n\n" "WITH\n" @@ -796,6 +803,7 @@ def test_given_append_cursor_model_when_compiling_then_sql_uses_expected_lower_b db_path: Path = tmp_path / "append_cursor.duckdb" model_header: str = ( "MODEL (\n" + ' description "Test model orders.",\n' " materialized incremental,\n" " incremental_strategy append,\n" " cursor ordered_at,\n" @@ -812,7 +820,7 @@ def test_given_append_cursor_model_when_compiling_then_sql_uses_expected_lower_b f'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{db_path}"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "models/orders.sql": model_header, }, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index 387bb23a98..f9d5697d63 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -44,7 +44,7 @@ def test_given_attached_measurement_audit_when_compiling_then_full_contract_reac 'database = ":memory:"\n' ), "models/orders.sql": """ - MODEL ( + MODEL (description "Test model orders.", materialized table, audits [ valid_order_rate ( @@ -138,7 +138,7 @@ def test_given_generic_measurement_definition_with_policy_when_compiling_then_de tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (audits [rate (thresholds (warn (below 100)))]); SELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.', audits [rate (thresholds (warn (below 100)))]); SELECT 1", "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate, thresholds (warn (below 100))); MEASURE (SELECT 100 AS rate FROM @relation); @@ -169,7 +169,7 @@ def test_given_measurement_attachment_without_thresholds_when_compiling_then_pol tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (audits [rate]); SELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.', audits [rate]); SELECT 1", "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate); MEASURE (SELECT 100 AS rate FROM @relation); @@ -201,7 +201,7 @@ def test_given_measurement_attachment_with_severity_when_compiling_then_conflict { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (audits [rate (severity warn, thresholds (warn (below 100)))]); SELECT 1" + "MODEL (description 'Test model orders.', audits [rate (severity warn, thresholds (warn (below 100)))]); SELECT 1" ), "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate); @@ -234,8 +234,8 @@ def test_given_standalone_measurement_with_header_policy_when_compiling_then_it_ tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\nSELECT 1 AS order_id", - "models/customers.sql": "MODEL ();\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id", + "models/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id", "audits/singular/rate.sql": """ AUDIT ( evaluation measurement, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 504f4cc9d9..8116e86214 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -46,7 +46,8 @@ def test_given_quoted_case_semantics_when_compiling_then_resolves_native_aliases ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql ) main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -77,7 +78,9 @@ def test_given_quoted_typo_when_validating_then_keeps_authored_spelling_and_span 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n' ) (tmp_path / "models").mkdir() - sql: str = "MODEL (database warehouse, schema analytics);\n" + test_case.sql + sql: str = ( + "MODEL (description 'Test model.', database warehouse, schema analytics);\n" + test_case.sql + ) (tmp_path / "models/orders.sql").write_text(sql) assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 payload: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -105,9 +108,12 @@ def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_ ) (tmp_path / "models").mkdir() (tmp_path / "sources").mkdir() - (tmp_path / "sources/orders.yml").write_text("sources:\n - name: orders\n table: orders\n") + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n" + ) (tmp_path / "models/report.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', database warehouse, schema analytics);\n" + + test_case.sql ) project: CompiledProject = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() @@ -164,7 +170,8 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql ) def repeated_binding(**kwargs: Any) -> None: @@ -211,7 +218,7 @@ def test_given_target_cache_policy_when_compiling_twice_then_reuses_only_enabled (tmp_path / "models").mkdir() model: Path = tmp_path / "models/orders.sql" model.write_text( - 'MODEL (database warehouse, schema analytics);\nSELECT SUM("Active") FROM (SELECT TRUE AS "Active") q' + 'MODEL (description "Test model orders.", database warehouse, schema analytics);\nSELECT SUM("Active") FROM (SELECT TRUE AS "Active") q' ) args: list[str] = [ "--project-dir", @@ -261,11 +268,12 @@ def test_given_quoted_catalog_binding_when_compiling_then_preserves_bound_expres ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS amount\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS id, 2 AS amount\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics); " + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics); " + + test_case.sql ) assert ( main( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index c7e54f315a..6503f5f1e2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -25,19 +25,19 @@ ) _PROJECT_FILES: dict[str, str] = { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 1}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), - "models/order_marker.sql": "MODEL (materialized table);\nSELECT 1 AS n\n", + "models/order_marker.sql": "MODEL (description 'Test model order_marker.', materialized table);\nSELECT 1 AS n\n", "python/tasks/count.py": ( "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" '@task(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" - " ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" + " '''Test task count_task.'''\n ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" ), } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py index 6a293153d9..bf704a1dc1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py @@ -49,11 +49,13 @@ def test_given_open_sources_when_validating_clauses_then_only_closed_shapes_prov ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n" - " - name: known\n table: known\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n - name: customers\n description: Test source customers.\n table: customers\n" + " - name: known\n description: Test source known.\n table: known\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n" ) (tmp_path / "models").mkdir() - (tmp_path / "models/report.sql").write_text("MODEL (materialized view);\n" + test_case.sql) + (tmp_path / "models/report.sql").write_text( + "MODEL (description 'Test model report.', materialized view);\n" + test_case.sql + ) result: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes @@ -102,13 +104,15 @@ def test_given_nested_function_calls_when_compiling_then_native_normalization_do functions: Path = tmp_path / "functions/sql" functions.mkdir(parents=True) (functions / "table_fn__orders.sql").write_text( - "FUNCTION (arguments (x INTEGER), returns table (id INTEGER)); SELECT x AS id" + "FUNCTION (description 'Test function table_fn__orders.', arguments (x INTEGER), returns table (id INTEGER)); SELECT x AS id" ) (functions / "udf__quantity.sql").write_text( - "FUNCTION (arguments (x INTEGER), returns INTEGER); x + 1" + "FUNCTION (description 'Test function udf__quantity.', arguments (x INTEGER), returns INTEGER); x + 1" ) (tmp_path / "models").mkdir() - (tmp_path / "models/orders.sql").write_text("MODEL (materialized view); " + test_case.sql) + (tmp_path / "models/orders.sql").write_text( + "MODEL (description 'Test model orders.', materialized view); " + test_case.sql + ) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes @@ -150,7 +154,7 @@ def without_deferred_requests(**kwargs: Any) -> Any: f'name = "orders"\nadapter = "{test_case.dialect}"\n[rules]\nselect = []\n' ) (tmp_path / "models").mkdir() - header: str = "MODEL (materialized view, database warehouse, schema analytics);\n" + header: str = "MODEL (description 'Test model.', materialized view, database warehouse, schema analytics);\n" (tmp_path / "models/orders.sql").write_text(header + f"SELECT 1 AS {test_case.projection}") (tmp_path / "models/report.sql").write_text( header + f'SELECT {test_case.reference} FROM __ref("orders")' @@ -166,15 +170,15 @@ def without_deferred_requests(**kwargs: Any) -> Any: [ NativeCatalogCase( "accepted numeric strings", - 'MODEL (materialized view, columns (quantity (audits [accepted_values (values ["1", "2"])]))); SELECT 1 AS quantity', + 'MODEL (description "Test model.", materialized view, columns (quantity (audits [accepted_values (values ["1", "2"])]))); SELECT 1 AS quantity', ), NativeCatalogCase( "relationship case", - 'MODEL (materialized view, columns (customer_id (audits [relationships (to __ref("customers"), field customer_id)]))); SELECT 1 AS customer_id', + 'MODEL (description "Test model.", materialized view, columns (customer_id (audits [relationships (to __ref("customers"), field customer_id)]))); SELECT 1 AS customer_id', ), NativeCatalogCase( "relationship coercion", - "MODEL (materialized view, columns (customer_id (audits [relationships (to __ref(\"customers\"), field customer_id)]))); SELECT '1' AS customer_id", + "MODEL (description 'Test model.', materialized view, columns (customer_id (audits [relationships (to __ref(\"customers\"), field customer_id)]))); SELECT '1' AS customer_id", ), ], ids=lambda case: case.description, @@ -190,7 +194,7 @@ def test_given_metadata_predicates_when_compiling_then_allows_dialect_coercion_a (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text(test_case.sql) (tmp_path / "models/customers.sql").write_text( - "MODEL (materialized view); SELECT 1 AS CUSTOMER_ID" + "MODEL (description 'Test model customers.', materialized view); SELECT 1 AS CUSTOMER_ID" ) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py index 550ec61f2c..9f6331740c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py @@ -20,13 +20,13 @@ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( "sources:\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" ), "models/stg_payments.sql": ( - "MODEL (materialized table, audits [orders_backed]);\n\n" + "MODEL (description 'Test model stg_payments.', materialized table, audits [orders_backed]);\n\n" 'SELECT payment_id FROM __source("raw_payments")' ), "audits/generic/orders_backed.sql": ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py index 034fd936a3..809ffa7432 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py @@ -16,7 +16,7 @@ ) from tests.integration.src.sqlbuild.compiler.pipeline.helpers import write_semantic_binding_project -_AUTHORITATIVE_MODEL: str = """MODEL ( +_AUTHORITATIVE_MODEL: str = """MODEL (description "Test model.", materialized table contract enforced columns ( @@ -114,7 +114,7 @@ def test_given_complete_contract_when_clause_references_missing_column_then_comp write_semantic_binding_project( project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, - downstream_sql=f"MODEL (materialized view);\n{test_case.query_sql}\n", + downstream_sql=f"MODEL (description 'Test model.', materialized view);\n{test_case.query_sql}\n", ) exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile", "--no-cache"]) @@ -150,7 +150,7 @@ def test_given_two_complete_contracts_when_unqualified_column_is_ambiguous_then_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("upstream") l CROSS JOIN __ref("other") r\n' ), ) @@ -184,9 +184,9 @@ def test_given_explicit_projection_when_output_column_is_missing_then_compile_fa ) -> None: write_semantic_binding_project( project_dir=tmp_path, - upstream_sql="MODEL (materialized table);\nSELECT id FROM raw_orders\n", + upstream_sql="MODEL (description 'Test model.', materialized table);\nSELECT id FROM raw_orders\n", downstream_sql=( - 'MODEL (materialized view);\nSELECT missing_column FROM __ref("upstream")\n' + 'MODEL (description "Test model.", materialized view);\nSELECT missing_column FROM __ref("upstream")\n' ), ) @@ -233,6 +233,7 @@ def test_given_source_contract_default_when_compiling_then_effective_policy_cont _ = (sources_dir / "raw.yml").write_text( f"""sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders {test_case.source_contract_yaml} columns: @@ -242,7 +243,7 @@ def test_given_source_contract_default_when_compiling_then_effective_policy_cont encoding="utf-8", ) _ = (models_dir / "downstream.sql").write_text( - "MODEL (materialized view, columns (missing (type INTEGER)));\n" + "MODEL (description 'Test model downstream.', materialized view, columns (missing (type INTEGER)));\n" 'SELECT CAST(missing AS INTEGER) AS missing FROM __source("raw_orders")\n', encoding="utf-8", ) @@ -274,12 +275,12 @@ def test_given_complete_inferred_intermediate_when_downstream_column_is_missing_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("intermediate") WHERE missing_derived = TRUE\n' ), ) _ = (tmp_path / "models" / "intermediate.sql").write_text( - 'MODEL (materialized view);\nSELECT id FROM __ref("upstream")\n', + 'MODEL (description "Test model intermediate.", materialized view);\nSELECT id FROM __ref("upstream")\n', encoding="utf-8", ) @@ -309,7 +310,7 @@ def test_given_model_analysis_disabled_when_column_is_missing_then_binding_is_sk project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " sql_analysis false\n" " contract enforced\n" @@ -349,12 +350,12 @@ def test_given_cached_dependent_error_when_upstream_analysis_disabled_then_stale project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - 'MODEL (materialized view);\nSELECT runtime_column FROM __ref("intermediate")\n' + 'MODEL (description "Test model.", materialized view);\nSELECT runtime_column FROM __ref("intermediate")\n' ), ) intermediate_path: Path = tmp_path / "models" / "intermediate.sql" _ = intermediate_path.write_text( - 'MODEL (materialized view);\nSELECT id FROM __ref("upstream")\n', + 'MODEL (description "Test model intermediate.", materialized view);\nSELECT id FROM __ref("upstream")\n', encoding="utf-8", ) @@ -364,7 +365,7 @@ def test_given_cached_dependent_error_when_upstream_analysis_disabled_then_stale assert "error[B002]: Unknown column 'runtime_column'" in initial_output _ = intermediate_path.write_text( - "MODEL (materialized view, sql_analysis false);\n" + "MODEL (description 'Test model.', materialized view, sql_analysis false);\n" 'SELECT runtime_column FROM __ref("upstream")\n', encoding="utf-8", ) @@ -401,7 +402,7 @@ def test_given_cli_analysis_disabled_when_sql_is_unsupported_then_compile_succee models_dir: Path = tmp_path / "models" models_dir.mkdir() _ = (models_dir / "vendor.sql").write_text( - "MODEL (materialized view);\nSELEC unsupported vendor sql\n", + "MODEL (description 'Test model vendor.', materialized view);\nSELEC unsupported vendor sql\n", encoding="utf-8", ) @@ -438,7 +439,7 @@ def test_given_lineage_uses_requested_when_query_filters_then_json_keeps_predica project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("upstream") u ' "WHERE u.category = 'active' ORDER BY u.id\n" ), @@ -491,7 +492,7 @@ def test_given_valid_using_and_qualify_alias_when_binding_then_compile_succeeds( project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" "SELECT ROW_NUMBER() OVER (ORDER BY l.id) AS rn " 'FROM __ref("upstream") l JOIN __ref("upstream") r USING (id) ' "QUALIFY rn = 1\n" @@ -524,7 +525,7 @@ def test_given_unresolved_star_when_contract_has_additional_column_then_compile_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " contract enforced\n" " columns (\n" @@ -563,7 +564,7 @@ def test_given_sized_varchar_cast_when_compiling_enforced_contract_then_type_mat project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " contract enforced\n" " columns (category (type VARCHAR(3)))\n" @@ -608,6 +609,7 @@ def test_given_untyped_source_when_cast_flows_through_ctes_then_declared_type_ma _ = (sources_dir / "raw.yml").write_text( """sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders columns: @@ -616,7 +618,7 @@ def test_given_untyped_source_when_cast_flows_through_ctes_then_declared_type_ma encoding="utf-8", ) model_path: Path = models_dir / "orders.sql" - model_sql: str = """MODEL ( + model_sql: str = """MODEL (description "Test model orders.", materialized view database analytics schema analytics @@ -693,7 +695,7 @@ def test_given_typed_union_ctes_when_compiling_enforced_contract_then_types_are_ " materialized table\n database analytics\n schema analytics\n", ), downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " database analytics\n" " schema analytics\n" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py index ab772d325c..f54b291d9a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -69,7 +69,7 @@ extra_files=( ( "models/independent.sql", - 'MODEL (materialized view);\nSELECT w.missing FROM __seed("waffle_types") w CROSS JOIN __ref("fact_orders") o\n', + 'MODEL (description "Test model independent.", materialized view);\nSELECT w.missing FROM __seed("waffle_types") w CROSS JOIN __ref("fact_orders") o\n', ), ), expected_errors=2, @@ -88,14 +88,17 @@ "predicate error does not poison another input projection", (), extra_files=( - ("models/available.sql", "MODEL (materialized view); SELECT 1 AS qty"), + ( + "models/available.sql", + "MODEL (description 'Test model available.', materialized view); SELECT 1 AS qty", + ), ( "models/predicate_bad.sql", - 'MODEL (materialized view); SELECT a.qty FROM __ref("available") a CROSS JOIN __ref("stg_orders") o WHERE o.qty > 5', + 'MODEL (description "Test model predicate_bad.", materialized view); SELECT a.qty FROM __ref("available") a CROSS JOIN __ref("stg_orders") o WHERE o.qty > 5', ), ( "models/independent_use.sql", - 'MODEL (materialized view); SELECT quantity FROM __ref("predicate_bad")', + 'MODEL (description "Test model independent_use.", materialized view); SELECT quantity FROM __ref("predicate_bad")', ), ), expected_errors=2, @@ -110,11 +113,11 @@ extra_files=( ( "sources/optional.yml", - "sources:\n - name: external_orders\n table: external_orders\n", + "sources:\n - name: external_orders\n description: Test source external_orders.\n table: external_orders\n", ), ( "models/open_read.sql", - 'MODEL (materialized view);\nSELECT id FROM __source("external_orders")\n', + 'MODEL (description "Test model open_read.", materialized view);\nSELECT id FROM __source("external_orders")\n', ), ), expected_fragments=( @@ -139,12 +142,12 @@ extra_files=( ( "models/wide.sql", - "MODEL (materialized view);\nSELECT 1 AS quantity, " + "MODEL (description 'Test model wide.', materialized view);\nSELECT 1 AS quantity, " + ", ".join(f"1 AS field_{index}" for index in range(20)), ), ( "models/wide_bad.sql", - 'MODEL (materialized view);\nSELECT w.qty FROM __ref("wide") w', + 'MODEL (description "Test model wide_bad.", materialized view);\nSELECT w.qty FROM __ref("wide") w', ), ), expected_fragments=("wide has: quantity", "and 11 more", "did you mean 'quantity'?"), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py index 9641558072..96e61e1311 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py @@ -75,7 +75,7 @@ _UPSTREAM, "SELECT 1 AS id", "B300", - "MODEL (materialized table, unique_key [missing]);\n", + "MODEL (description 'Test model.', materialized table, unique_key [missing]);\n", ), SemanticCompileCase( "date literal valid control", @@ -117,16 +117,16 @@ def test_given_semantic_scope_when_compiling_then_closed_shapes_are_checked( ) -> None: write_semantic_binding_project( project_dir=tmp_path, - upstream_sql="MODEL (materialized view);\n" + test_case.upstream, + upstream_sql="MODEL (description 'Test model.', materialized view);\n" + test_case.upstream, downstream_sql=test_case.header + test_case.downstream, ) (tmp_path / "sources").mkdir(exist_ok=True) (tmp_path / "sources" / "orders.yml").write_text( "sources:\n" - " - name: open_orders\n table: orders\n" - " - name: closed_orders\n table: orders\n contract: enforced\n" + " - name: open_orders\n description: Test source open_orders.\n table: orders\n" + " - name: closed_orders\n description: Test source closed_orders.\n table: orders\n contract: enforced\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: expression_orders\n expression: SELECT 1 AS id\n", + " - name: expression_orders\n description: Test source expression_orders.\n expression: SELECT 1 AS id\n", encoding="utf-8", ) exit_code: int = main( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index 342e9704bd..ff4cb8db7c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -91,7 +91,7 @@ def test_given_snowflake_semantics_when_compiling_then_preserves_native_verdicts (tmp_path / "models").mkdir() (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n" " - {name: order_id, type: INTEGER}\n" " - {name: quantity, type: INTEGER}\n" " - {name: value, type: VARCHAR}\n" @@ -99,7 +99,8 @@ def test_given_snowflake_semantics_when_compiling_then_preserves_native_verdicts " - {name: ordered_at, type: TIMESTAMP}\n" ) (tmp_path / "models/report.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', materialized view, database warehouse, schema analytics);\n" + + test_case.sql ) assert ( main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) @@ -135,7 +136,8 @@ def test_given_snowflake_query_when_inferring_rich_types_then_preserves_proven_o (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') (tmp_path / "models").mkdir() (tmp_path / "models/report.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', materialized view, database warehouse, schema analytics);\n" + + test_case.sql ) project: CompiledProject = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index b96ee287a2..e0be6124d6 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -44,11 +44,11 @@ def test_given_cursor_families_when_compiling_then_accepts_temporal_pairs_only( (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n" f" columns:\n - name: ordered_at\n type: {test_case.column_type}\n" ) (tmp_path / "models/recent_orders.sql").write_text( - "MODEL (materialized incremental, database warehouse, schema analytics, " + "MODEL (description 'Test model recent_orders.', materialized incremental, database warehouse, schema analytics, " f"incremental_strategy append, cursor ordered_at, cursor_type {test_case.cursor_type}, " 'cursor_grain day);\nSELECT ordered_at FROM __source("orders")' ) @@ -83,11 +83,11 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n" " columns:\n - name: customer\n type: VARCHAR\n" ) (tmp_path / "models/quoted_orders.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + "MODEL (description 'Test model quoted_orders.', materialized view, database warehouse, schema analytics);\n" 'SELECT o."customer" FROM __source("orders") o' ) for _ in range(2): @@ -138,21 +138,21 @@ def test_given_type_error_root_when_binding_dependants_then_recovers_only_poison ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), " + "MODEL (description 'Test model orders.', materialized view, contract enforced, columns (quantity (type INTEGER), " "ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, " "CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" ) (tmp_path / "models/broken.sql").write_text( - f"MODEL (materialized view{test_case.root_contract});\nSELECT quantity + ordered_at AS total, " + f"MODEL (description 'Test model broken.', materialized view{test_case.root_contract});\nSELECT quantity + ordered_at AS total, " 'ordered_at FROM __ref("orders")' ) (tmp_path / "models/downstream.sql").write_text( - "MODEL (materialized view);\nSELECT total > ordered_at AS compared" + "MODEL (description 'Test model downstream.', materialized view);\nSELECT total > ordered_at AS compared" + test_case.independent_sql + ' FROM __ref("broken")' ) (tmp_path / "models/final_orders.sql").write_text( - 'MODEL (materialized view);\nSELECT compared AS final_value FROM __ref("downstream")' + 'MODEL (description "Test model final_orders.", materialized view);\nSELECT compared AS final_value FROM __ref("downstream")' ) main(["--project-dir", str(tmp_path), "compile", "--json"]) result: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -181,11 +181,11 @@ def test_given_repetitive_sql_when_mapping_diagnostic_then_preserves_authored_sp (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view);\nSELECT CAST(1 AS INTEGER) AS quantity" + "MODEL (description 'Test model orders.', materialized view);\nSELECT CAST(1 AS INTEGER) AS quantity" ) projections: str = ",\n".join(f"quantity + {index} AS amount_{index}" for index in range(300)) sql: str = ( - "MODEL (materialized view);\nSELECT\n" + "MODEL (description 'Test model s.', materialized view);\nSELECT\n" + projections + "\nFROM __ref(\"orders\")\nWHERE quantity > TIMESTAMP '2026-04-01'" ) @@ -216,10 +216,10 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() source: Path = tmp_path / "sources/orders.yml" - declaration: str = "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n - name: quantity\n type: INTEGER\n" + declaration: str = "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n - name: quantity\n type: INTEGER\n" source.write_text(declaration) (tmp_path / "models/totals.sql").write_text( - 'MODEL (materialized view);\nSELECT quantity + 1 AS total FROM __source("orders")' + 'MODEL (description "Test model totals.", materialized view);\nSELECT quantity + 1 AS total FROM __source("orders")' ) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] with monkeypatch.context() as patch: @@ -258,10 +258,10 @@ def test_given_repeated_root_messages_when_recovering_then_preserves_each_output ) (tmp_path / "models").mkdir() files: dict[str, str] = { - "orders": "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", - "broken": 'MODEL (materialized view, contract enforced, columns (total_a (type INTEGER), total_b (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT quantity + ordered_at AS total_a, quantity + ordered_at AS total_b, ordered_at FROM __ref("orders")', - "downstream": 'MODEL (materialized view);\nSELECT total_a > ordered_at AS a, total_b > ordered_at AS b FROM __ref("broken")', - "final_orders": 'MODEL (materialized view);\nSELECT a, b FROM __ref("downstream")', + "orders": "MODEL (description 'Test model.', materialized view, contract enforced, columns (quantity (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", + "broken": 'MODEL (description "Test model.", materialized view, contract enforced, columns (total_a (type INTEGER), total_b (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT quantity + ordered_at AS total_a, quantity + ordered_at AS total_b, ordered_at FROM __ref("orders")', + "downstream": 'MODEL (description "Test model.", materialized view);\nSELECT total_a > ordered_at AS a, total_b > ordered_at AS b FROM __ref("broken")', + "final_orders": 'MODEL (description "Test model.", materialized view);\nSELECT a, b FROM __ref("downstream")', } for name, sql in files.items(): (tmp_path / "models" / f"{name}.sql").write_text(sql) @@ -322,20 +322,20 @@ def test_given_udf_argument_columns_when_compiling_then_checks_declared_argument (tmp_path / "functions/sql").mkdir(parents=True) (tmp_path / "sources/raw.yml").write_text( "sources:\n" - " - name: orders\n table: orders\n contract: enforced\n columns:\n" + " - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n" " - name: id\n type: INTEGER\n" " - name: customer_id\n type: INTEGER\n" " - name: status\n type: VARCHAR\n" " - name: amount\n type: DOUBLE\n" - " - name: customers\n table: customers\n contract: enforced\n columns:\n" + " - name: customers\n description: Test source customers.\n table: customers\n contract: enforced\n columns:\n" " - name: id\n type: INTEGER\n" " - name: customer_name\n type: VARCHAR\n" ) (tmp_path / "functions/sql/add_one.sql").write_text( - "FUNCTION (\n arguments (input_value DOUBLE),\n returns DOUBLE,\n);\n\ninput_value + 1\n" + "FUNCTION (description 'Test function add_one.',\n arguments (input_value DOUBLE),\n returns DOUBLE,\n);\n\ninput_value + 1\n" ) (tmp_path / "models/order_totals.sql").write_text( - f"MODEL (materialized table);\n\nSELECT {test_case.projection}\n" + f"MODEL (description 'Test model order_totals.', materialized table);\n\nSELECT {test_case.projection}\n" 'FROM __source("orders") o JOIN __source("customers") c ON o.customer_id = c.id\n' ) main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index b2b49e39e4..c794065571 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -57,7 +57,8 @@ def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - f"MODEL (materialized table);\n{test_case.query_sql}", encoding="utf-8" + f"MODEL (description 'Test model orders.', materialized table);\n{test_case.query_sql}", + encoding="utf-8", ) project: CompiledProject = compile_project( @@ -95,19 +96,19 @@ def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model orders.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" "SELECT CAST(2.5 AS DOUBLE) AS amount", encoding="utf-8", ) (models / "combined_orders.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model combined_orders.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" 'WITH combined AS (SELECT CAST(amount AS DOUBLE) AS amount FROM __ref("orders") ' 'UNION ALL SELECT list_extract([amount], 1) AS amount FROM __ref("refunds")) ' "SELECT amount FROM combined", encoding="utf-8", ) (models / "refunds.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model refunds.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" "SELECT CAST(1.25 AS DOUBLE) AS amount", encoding="utf-8", ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py index 08d36c33c2..cd1d1b2cd1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py @@ -20,53 +20,53 @@ [ SharedBindingQueryCase( description="identical shapes and inputs share one query", - orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', - customers_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("customers")', + orders_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', + customers_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers")', expected_shared_queries=1, ), SharedBindingQueryCase( description="a relation name used as a qualifier is not shared", orders_summary_sql=( - 'MODEL (materialized view); SELECT orders.id FROM __ref("orders") AS orders' + 'MODEL (description "Test model.", materialized view); SELECT orders.id FROM __ref("orders") AS orders' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT customers.id FROM __ref("customers") AS customers' + 'MODEL (description "Test model.", materialized view); SELECT customers.id FROM __ref("customers") AS customers' ), expected_shared_queries=0, ), SharedBindingQueryCase( description="an authored lowercase stub-named CTE is not shared", orders_summary_sql=( - "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __sqlbuild_project_input_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' ), customers_summary_sql=( - "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __sqlbuild_project_input_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' ), expected_shared_queries=0, later_models=( ( "next_orders", - 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', ), ), ), SharedBindingQueryCase( description="an authored uppercase stub-named CTE is not shared", orders_summary_sql=( - "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' ), customers_summary_sql=( - "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' ), expected_shared_queries=0, later_models=( ( "next_orders", - 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', ), ), ), @@ -116,10 +116,10 @@ def test_given_models_with_equal_shapes_when_compiling_then_shares_only_lossless SharedBindingQueryCase( description="findings after the relation keep each model's names and positions", orders_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' ), expected_shared_queries=1, expected_codes=("B002",), @@ -163,19 +163,19 @@ def test_given_shared_query_with_finding_when_compiling_then_reports_each_model_ [ SharedBindingQueryCase( description="later-wave members reuse an exact shared result", - orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', + orders_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', customers_summary_sql=( - 'MODEL (materialized view); SELECT id, amount FROM __ref("customers")' + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers")' ), expected_shared_queries=1, later_models=( ( "orders_rollup", - 'MODEL (materialized view); SELECT id, amount FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders_summary")', ), ( "customers_rollup", - 'MODEL (materialized view); SELECT id, amount FROM __ref("customers_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers_summary")', ), ), expected_lineage=( @@ -235,18 +235,21 @@ def test_given_later_wave_members_with_equal_keys_when_compiling_then_reuses_sha SharedBindingQueryCase( description="a later-wave member of a group with findings keeps its own names", orders_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' ), expected_shared_queries=2, expected_codes=("B002",), later_models=( - ("orders_mid", 'MODEL (materialized view); SELECT id, amount FROM __ref("orders")'), + ( + "orders_mid", + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', + ), ( "orders_late", - 'MODEL (materialized view); SELECT id FROM __ref("orders_mid") WHERE missing > 0', + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders_mid") WHERE missing > 0', ), ), expected_findings=( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index 9ba7d3c23c..3d3b9ece48 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -30,10 +30,10 @@ def test_given_expanded_mixed_mode_tests_when_compiling_project_then_complete_in tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", "models/_constants/order_id.sql": "CONSTANT (name order_id, value 7);\n", "functions/sql/increment.sql": ( - "FUNCTION (arguments (value INTEGER), returns INTEGER);\n\nvalue + 1\n" + "FUNCTION (description 'Test function increment.', arguments (value INTEGER), returns INTEGER);\n\nvalue + 1\n" ), "tests/unit/a_orders.sql": ( "TEST (name orders_match);\n\n" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 8d0cee809f..dfbfa47666 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -59,9 +59,11 @@ def test_given_parameterized_model_test_when_compiled_and_executed_then_duckdb_p { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "sources/raw.yml": ( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n'), "tests/unit/orders.sql": f""" TEST ( name "parameterized_orders", diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 2542323465..82d1a51f31 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -42,11 +42,11 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir 'database = ":memory:"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\nSELECT order_id, status FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 'open' AS status\n" " columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" @@ -109,12 +109,12 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ 'database = ":memory:"\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT TRY_CAST(status AS DOUBLE) AS parsed_status " + "MODEL (description 'Test model orders.');\n\nSELECT TRY_CAST(status AS DOUBLE) AS parsed_status " 'FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT '1.5' AS status\n" " contract: enforced\n" " columns:\n" diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 5d4b27c89f..7c7fde33f7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -595,11 +595,13 @@ def plan_source_reading_project( project_files: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "sources/raw.yml": ( - f"sources:\n - name: raw_orders\n schema: {source_schema}\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + f" schema: {source_schema}\n" f" table: {source_table}\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), } relative_path: str diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py index c07fce1696..8d1810466c 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py @@ -39,10 +39,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/alpha.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alpha' AS name" + "MODEL (description 'Test model alpha.', materialized table);\n\nSELECT 1 AS id, 'alpha' AS name" ), "models/beta.sql": ( - "MODEL (materialized table);\n\nSELECT 2 AS id, 'beta' AS name" + "MODEL (description 'Test model beta.', materialized table);\n\nSELECT 2 AS id, 'beta' AS name" ), }, expected_status=BuildStatus.SUCCESS, @@ -64,18 +64,18 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/staging/stg_raw.sql": ( - "MODEL (materialized view);\n\nSELECT 10 AS id, 'row' AS val" + "MODEL (description 'Test model stg_raw.', materialized view);\n\nSELECT 10 AS id, 'row' AS val" ), "models/branch_a.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model branch_a.', materialized table);\n\n" "SELECT id, val || '_a' AS val FROM __ref(\"stg_raw\")" ), "models/branch_b.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model branch_b.', materialized table);\n\n" "SELECT id, val || '_b' AS val FROM __ref(\"stg_raw\")" ), "models/merged.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model merged.', materialized table);\n\n" "SELECT a.id, a.val AS a_val, b.val AS b_val " 'FROM __ref("branch_a") a ' 'JOIN __ref("branch_b") b ON a.id = b.id' @@ -95,12 +95,14 @@ max_concurrency=2, project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/layer_1.sql": ("MODEL (materialized table);\n\nSELECT 100 AS val"), + "models/layer_1.sql": ( + "MODEL (description 'Test model layer_1.', materialized table);\n\nSELECT 100 AS val" + ), "models/layer_2.sql": ( - 'MODEL (materialized table);\n\nSELECT val * 2 AS val FROM __ref("layer_1")' + 'MODEL (description "Test model layer_2.", materialized table);\n\nSELECT val * 2 AS val FROM __ref("layer_1")' ), "models/layer_3.sql": ( - 'MODEL (materialized table);\n\nSELECT val + 1 AS val FROM __ref("layer_2")' + 'MODEL (description "Test model layer_3.", materialized table);\n\nSELECT val + 1 AS val FROM __ref("layer_2")' ), }, expected_status=BuildStatus.SUCCESS, @@ -170,6 +172,7 @@ def test_given_concurrent_provider_backed_loaders_when_executing_then_share_prov class ConcurrentMarkerProvider(Provider): + '''Test provider ConcurrentMarkerProvider.''' marker_path: str = {str(marker_path)!r} @property @@ -197,12 +200,14 @@ def mark(self, label): @loader def raw_alpha(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_alpha.''' concurrent_marker_provider.mark("alpha") return [{"event_id": 1}] @loader def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_beta.''' concurrent_marker_provider.mark("beta") return [{"event_id": 2}] """ @@ -212,12 +217,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): """ sources: - name: raw_alpha + description: Test source raw_alpha. managed: true write_strategy: table columns: - name: event_id type: INTEGER - name: raw_beta + description: Test source raw_beta. managed: true write_strategy: table columns: @@ -227,10 +234,10 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): ).strip() + "\n", "models/fact_alpha.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' + 'MODEL (description "Test model fact_alpha.", materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' ), "models/fact_beta.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_beta")\n' + 'MODEL (description "Test model fact_beta.", materialized table);\n\nSELECT * FROM __source("raw_beta")\n' ), }, ) @@ -269,9 +276,11 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/healthy.sql": ( + "MODEL (description 'Test model healthy.', materialized table);\n\nSELECT 42 AS val" ), - "models/healthy.sql": ("MODEL (materialized table);\n\nSELECT 42 AS val"), }, expected_status=BuildStatus.FAILED, expected_success_count=1, @@ -291,12 +300,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model broken.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/downstream.sql": ( - 'MODEL (materialized table);\n\nSELECT 1 AS id FROM __ref("broken")' + 'MODEL (description "Test model downstream.", materialized table);\n\nSELECT 1 AS id FROM __ref("broken")' + ), + "models/independent.sql": ( + "MODEL (description 'Test model independent.', materialized table);\n\nSELECT 99 AS val" ), - "models/independent.sql": ("MODEL (materialized table);\n\nSELECT 99 AS val"), }, expected_status=BuildStatus.FAILED, expected_success_count=1, @@ -319,9 +330,11 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/aaa_broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model aaa_broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/zzz_independent.sql": ( + "MODEL (description 'Test model zzz_independent.', materialized table);\n\nSELECT 1 AS id" ), - "models/zzz_independent.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), }, fail_fast=True, expected_status=BuildStatus.FAILED, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 98193b4a7e..272150769b 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -118,6 +118,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -125,7 +126,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -150,7 +151,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( + "\n", "models/shipments.sql": dedent( """ - MODEL ( + MODEL (description "Test model shipments.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 897b8acf68..9785765a12 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -52,16 +52,20 @@ max_concurrency=3, project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/staging/stg_a.sql": ("MODEL (materialized view);\n\nSELECT 1 AS id"), - "models/staging/stg_b.sql": ("MODEL (materialized view);\n\nSELECT 2 AS id"), + "models/staging/stg_a.sql": ( + "MODEL (description 'Test model stg_a.', materialized view);\n\nSELECT 1 AS id" + ), + "models/staging/stg_b.sql": ( + "MODEL (description 'Test model stg_b.', materialized view);\n\nSELECT 2 AS id" + ), "models/mid_a.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_a")' + 'MODEL (description "Test model mid_a.", materialized table);\n\nSELECT id FROM __ref("stg_a")' ), "models/mid_b.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_b")' + 'MODEL (description "Test model mid_b.", materialized table);\n\nSELECT id FROM __ref("stg_b")' ), "models/final.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model final.', materialized table);\n\n" "SELECT a.id AS a_id, b.id AS b_id " 'FROM __ref("mid_a") a ' 'CROSS JOIN __ref("mid_b") b' diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py index 18b9028045..83944e2890 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py @@ -78,6 +78,7 @@ def test_given_sequential_microbatch_when_repeated_then_uses_only_physical_water """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -85,7 +86,7 @@ def test_given_sequential_microbatch_when_repeated_then_uses_only_physical_water + "\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py index 22ed209e90..06f7212128 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py @@ -302,10 +302,10 @@ def test_given_custom_materialization_when_audit_finds_rows_in_staging_then_bloc project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/regular.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'regular' AS name" + "MODEL (description 'Test model regular.', materialized table);\n\nSELECT 1 AS id, 'regular' AS name" ), "models/custom_model.sql": ( - "MODEL (materialized test_custom);\n\nSELECT 2 AS id, 'custom' AS name" + "MODEL (description 'Test model custom_model.', materialized test_custom);\n\nSELECT 2 AS id, 'custom' AS name" ), }, expected_status=BuildStatus.SUCCESS, @@ -323,10 +323,10 @@ def test_given_custom_materialization_when_audit_finds_rows_in_staging_then_bloc project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/upstream.sql": ( - "MODEL (materialized table);\n\nSELECT 10 AS id, 'upstream' AS origin" + "MODEL (description 'Test model upstream.', materialized table);\n\nSELECT 10 AS id, 'upstream' AS origin" ), "models/downstream_custom.sql": ( - 'MODEL (materialized test_custom);\n\nSELECT id, origin FROM __ref("upstream")' + 'MODEL (description "Test model downstream_custom.", materialized test_custom);\n\nSELECT id, origin FROM __ref("upstream")' ), }, expected_status=BuildStatus.SUCCESS, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 05e2bb5231..e515555b70 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -50,7 +50,7 @@ def capped_dependency_producer_sql(*, action: MicrobatchLimitAction | None) -> s limit_sql: str = limit_blocks[action] return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -81,7 +81,7 @@ def capped_dependency_consumer_sql( return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -108,7 +108,7 @@ def capped_filter_consumer_sql(*, input_name: str = "capped_events") -> str: return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -136,7 +136,7 @@ def plain_capped_consumer_sql(*, materialized: str) -> str: return dedent( f""" - MODEL (materialized {materialized}); + MODEL (description "Test model.", materialized {materialized}); SELECT id, event_time FROM __ref("capped_events") """ ) @@ -147,7 +147,7 @@ def dependency_view_sql(*, upstream_name: str) -> str: return dedent( f""" - MODEL (materialized view); + MODEL (description "Test model.", materialized view); SELECT id, event_time FROM __ref("{upstream_name}") """ ) @@ -177,7 +177,7 @@ def capped_microbatch_intermediary_sql(*, input_role: str) -> str: } return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -215,7 +215,8 @@ def capped_dependency_warehouse_files(*, project_dir: Path) -> dict[str, str]: f"[connection]\ndatabase = {json.dumps(str(database_path))}\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" ), } diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index e0e158de04..491d2b3e30 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -426,7 +426,7 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " database: analytics\n" " schema: raw\n" " table: orders\n" @@ -439,14 +439,15 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: " type: NUMBER(10, 2)\n" ), "models/marts/orders_table.sql": ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model orders_table.',\n materialized table,\n);\n\n" 'SELECT order_id, ordered_at, amount FROM __source("raw_orders")\n' ), "models/marts/orders_view.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT order_id, amount FROM __ref("orders_table")\n' + 'MODEL (description "Test model orders_view.",' + '\n materialized view,\n);\n\nSELECT order_id, amount FROM __ref("orders_table")\n' ), "models/marts/orders_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [order_id],\n" @@ -459,7 +460,7 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: 'SELECT order_id, ordered_at, amount FROM __source("raw_orders")\n' ), "models/marts/orders_daily.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders_daily.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index c28eab2166..b565a28996 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -54,7 +54,7 @@ _RAW_ORDERS_WITH_NOT_NULL_AUDIT: str = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -64,10 +64,10 @@ ) _NOT_NULL_AUDIT: str = 'AUDIT ();\n\nSELECT @column FROM __ref("@model") WHERE @column IS NULL' _TABLE_WITH_ID_NOT_NULL_AUDIT: str = ( - "MODEL (materialized table, columns (id (audits [not_null])));\n\n" + "MODEL (description 'Test model.', materialized table, columns (id (audits [not_null])));\n\n" ) _VIEW_WITH_ID_NOT_NULL_AUDIT: str = ( - "MODEL (materialized view, columns (id (audits [not_null])));\n\n" + "MODEL (description 'Test model.', materialized view, columns (id (audits [not_null])));\n\n" ) @@ -104,14 +104,14 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "functions/sql/is_positive_int.sql": ( - "FUNCTION (\n" + "FUNCTION (description 'Test function is_positive_int.',\n" " arguments (a_string VARCHAR),\n" " returns BOOLEAN\n" ");\n\n" "regexp_matches(a_string, '^[0-9]+$')" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -143,10 +143,10 @@ " runtime_version='3.11',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -181,10 +181,10 @@ " runtime_version='3.11',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -209,9 +209,9 @@ description="two independent models both succeed", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name", "models/payments.sql": ( - "MODEL (materialized table);\n\nSELECT 10 AS payment_id, 500 AS amount" + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 10 AS payment_id, 500 AS amount" ), }, expected_status=BuildStatus.SUCCESS, @@ -230,10 +230,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 42 AS id, 'bob' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 42 AS id, 'bob' AS name" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" 'SELECT id, name FROM __ref("stg_orders") WHERE id = 42' ), }, @@ -261,7 +261,7 @@ 'schema = "dev_schema"\n' ), "sqlbuild_local.toml": 'target = "dev"\n', - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name", }, expected_status=BuildStatus.SUCCESS, expected_success_count=1, @@ -309,7 +309,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (id (audits [unique])));\n\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" "SELECT * FROM (VALUES (1), (NULL), (NULL)) AS input(id)" ), }, @@ -325,7 +325,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (status (audits [accepted_values " + "MODEL (description 'Test model orders.', materialized table, columns (status (audits [accepted_values " '(values ["placed", "completed"])])));\n\n' "SELECT * FROM (VALUES ('placed'), ('completed'), (NULL)) AS input(status)" ), @@ -344,9 +344,9 @@ description="built-in relationships audit allows matching referenced values", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/customers.sql": "MODEL (description 'Test model customers.', materialized table);\n\nSELECT 1 AS id", "models/orders.sql": ( - "MODEL (materialized table, columns (customer_id (audits [relationships " + "MODEL (description 'Test model orders.', materialized table, columns (customer_id (audits [relationships " '(to __ref("customers"), field id)])));\n\n' 'SELECT id AS customer_id FROM __ref("customers") UNION ALL SELECT NULL' ), @@ -379,7 +379,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML_WARN, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -394,8 +394,12 @@ description="end audit warn succeeds build with warning", project_files={ "sqlbuild_project.toml": _PROJECT_YML_WARN, - "models/orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 2 AS payment_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id" + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 2 AS payment_id" + ), "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' ), @@ -414,9 +418,11 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 1 AS payment_id" ), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 1 AS payment_id"), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' @@ -459,7 +465,7 @@ description="build writes fingerprints when query tracking is enabled", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id", }, expected_status=BuildStatus.SUCCESS, expected_success_count=1, @@ -516,10 +522,10 @@ def test_given_build_plan_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS id, 'alice' AS name" ), "models/dim_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' + 'MODEL (description "Test model dim_orders.", materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.SUCCESS, @@ -573,10 +579,10 @@ def test_given_view_build_plan_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/bad_view.sql": ( - "MODEL (materialized view);\n\nSELECT * FROM nonexistent_source_table" + "MODEL (description 'Test model bad_view.', materialized view);\n\nSELECT * FROM nonexistent_source_table" ), "models/downstream.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("bad_view")' + 'MODEL (description "Test model downstream.", materialized table);\n\nSELECT * FROM __ref("bad_view")' ), }, expected_status=BuildStatus.FAILED, @@ -656,10 +662,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -677,7 +683,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (id (type INTEGER)),\n" @@ -700,7 +706,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML_IMMEDIATE, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (id (type INTEGER)),\n" @@ -720,7 +726,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -752,7 +758,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " contract enforced,\n" " columns (\n" @@ -799,10 +805,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( " schema='udfs',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT __udf("is_positive_int")(' "'123'" ") AS is_positive" @@ -827,13 +833,13 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), "models/payments.sql": ( - "MODEL (materialized table);\n\nSELECT 99 AS payment_id, 750 AS amount" + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 99 AS payment_id, 750 AS amount" ), }, expected_status=BuildStatus.FAILED, @@ -868,7 +874,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (id (audits [unique])));\n\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" "SELECT * FROM (VALUES (1), (1), (NULL)) AS input(id)" ), }, @@ -883,7 +889,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (status (audits [accepted_values " + "MODEL (description 'Test model orders.', materialized table, columns (status (audits [accepted_values " '(values ["placed", "completed"])])));\n\n' "SELECT 'cancelled' AS status" ), @@ -898,9 +904,9 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( description="built-in relationships audit blocks missing referenced values", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/customers.sql": "MODEL (description 'Test model customers.', materialized table);\n\nSELECT 1 AS id", "models/orders.sql": ( - "MODEL (materialized table, columns (customer_id (audits [relationships " + "MODEL (description 'Test model orders.', materialized table, columns (customer_id (audits [relationships " '(to __ref("customers"), field id)])));\n\n' 'SELECT 2 AS customer_id FROM __ref("customers")' ), @@ -920,9 +926,11 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT 7 AS id, 'carol' AS name" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 7 AS id, 'carol' AS name" + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 1 AS payment_id" ), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 1 AS payment_id"), "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' ), @@ -940,9 +948,11 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/aaa_broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model aaa_broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/zzz_healthy.sql": ( + "MODEL (description 'Test model zzz_healthy.', materialized table);\n\nSELECT 1 AS id" ), - "models/zzz_healthy.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), }, fail_fast=True, expected_status=BuildStatus.FAILED, @@ -956,7 +966,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -973,10 +983,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -1009,12 +1019,12 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (\n materialized table\n" + "MODEL (description 'Test model stg_orders.',\n materialized table\n" " pre_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 1 AS id" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -1031,12 +1041,12 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (\n materialized table\n" + "MODEL (description 'Test model stg_orders.',\n materialized table\n" " post_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 88 AS id" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -1054,10 +1064,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken_a.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_a" + "MODEL (description 'Test model broken_a.', materialized table);\n\nSELECT * FROM nonexistent_a" ), "models/broken_b.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_b" + "MODEL (description 'Test model broken_b.', materialized table);\n\nSELECT * FROM nonexistent_b" ), }, expected_status=BuildStatus.FAILED, @@ -1115,7 +1125,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _PASSING_TEST_SQL, }, @@ -1130,7 +1140,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, @@ -1147,7 +1157,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, @@ -1163,10 +1173,10 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "models/dim_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' + 'MODEL (description "Test model dim_orders.", materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, diff --git a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py index 0463e6038c..8efa445fc4 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py @@ -37,7 +37,7 @@ def test_given_measurement_outcomes_when_building_then_gates_and_persists_canoni connection: Any, ) -> None: model_sql = """ - MODEL ( + MODEL (description "Test model.", materialized table, audits [ configured_rate ( diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index d853d2d81c..ebac3b9955 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -59,10 +59,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -109,12 +109,12 @@ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -171,10 +171,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -326,12 +326,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -390,12 +390,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_extracts\n" + " - name: raw_customer_region_extracts\n description: Test source raw_customer_region_extracts.\n" " schema: main\n" " table: raw_customer_region_extracts\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -451,12 +451,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -501,12 +501,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_changes\n" + " - name: raw_customer_changes\n description: Test source raw_customer_changes.\n" " schema: main\n" " table: raw_customer_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -569,12 +569,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_changes\n" + " - name: raw_customer_region_changes\n description: Test source raw_customer_region_changes.\n" " schema: main\n" " table: raw_customer_region_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -633,12 +633,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -753,10 +753,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -782,10 +782,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -807,10 +807,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -834,10 +834,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -862,12 +862,12 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -899,12 +899,12 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_changes\n" + " - name: raw_customer_changes\n description: Test source raw_customer_changes.\n" " schema: main\n" " table: raw_customer_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -985,10 +985,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1030,12 +1030,12 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1089,10 +1089,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1132,10 +1132,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1175,10 +1175,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1218,12 +1218,12 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1283,10 +1283,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1415,12 +1415,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1473,12 +1473,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_daily\n" + " - name: raw_customer_region_daily\n description: Test source raw_customer_region_daily.\n" " schema: main\n" " table: raw_customer_region_daily\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1532,12 +1532,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1577,12 +1577,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_daily\n" + " - name: raw_customer_region_daily\n description: Test source raw_customer_region_daily.\n" " schema: main\n" " table: raw_customer_region_daily\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1688,10 +1688,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1716,12 +1716,12 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1749,10 +1749,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1773,10 +1773,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1796,10 +1796,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1870,10 +1870,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -1915,10 +1915,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1956,10 +1956,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -1993,10 +1993,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2031,10 +2031,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2072,12 +2072,12 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -2168,11 +2168,11 @@ def test_given_current_state_snapshot_when_source_row_disappears_then_hard_delet initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2184,11 +2184,11 @@ def test_given_current_state_snapshot_when_source_row_disappears_then_hard_delet changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2273,11 +2273,11 @@ def test_given_snapshot_source_adds_column_when_building_then_appends_target_col initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2295,11 +2295,11 @@ def test_given_snapshot_source_adds_column_when_building_then_appends_target_col changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2391,11 +2391,11 @@ def test_given_contract_snapshot_adds_declared_column_when_building_then_require initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2413,11 +2413,11 @@ def test_given_contract_snapshot_adds_declared_column_when_building_then_require changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2510,11 +2510,11 @@ def test_given_confirmed_contract_snapshot_adds_declared_column_when_building_th initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2526,11 +2526,11 @@ def test_given_confirmed_contract_snapshot_adds_declared_column_when_building_th changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2606,11 +2606,11 @@ def test_given_wildcard_check_snapshot_source_adds_column_when_building_then_req initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2622,11 +2622,11 @@ def test_given_wildcard_check_snapshot_source_adds_column_when_building_then_req changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2710,12 +2710,12 @@ def test_given_confirmed_wildcard_schema_change_when_building_then_appends_targe "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2804,12 +2804,12 @@ def test_given_hard_delete_snapshot_when_duplicate_keys_fail_then_target_history "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2872,12 +2872,12 @@ def test_given_check_snapshot_with_initial_valid_from_when_building_then_uses_co "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2938,12 +2938,12 @@ def test_given_timestamp_snapshot_with_updated_at_initial_validity_when_building "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -3004,12 +3004,12 @@ def test_given_timestamp_snapshot_with_execution_initial_validity_when_building_ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -3072,12 +3072,12 @@ def test_given_check_snapshot_with_execution_initial_validity_when_building_then "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -3130,12 +3130,12 @@ def test_given_snapshot_initial_validity_config_when_planning_then_plan_entry_pr "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -3203,12 +3203,12 @@ def test_given_check_snapshot_with_final_audit_when_building_then_runs_audit( "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index 625fb41eb0..08d4ea960a 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -25,11 +25,11 @@ 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "demo.duckdb"\n' ), "models/customer_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_totals.', materialized table);\n\n" "SELECT customer_id, SUM(amount) AS total FROM raw_orders GROUP BY customer_id" ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " incremental_strategy append,\n" " cursor updated_at,\n" diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index b13418d2be..b7d7d40518 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -169,10 +169,11 @@ def build_authored_cte_project_files() -> dict[str, str]: "sql_analysis = true\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "WITH base AS (\n" " -- a comment with an unmatched ) parenthesis\n" " SELECT id AS order_id, amount, 'it''s (fine)' AS note /* ( */\n" @@ -182,7 +183,7 @@ def build_authored_cte_project_files() -> dict[str, str]: "SELECT order_id, amount, note, tag FROM tagged\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "WITH totals AS (\n" ' WITH ranked AS (SELECT order_id, amount, tag FROM __ref("stg_orders"))\n' " SELECT order_id, amount * 2 AS doubled, tag FROM ranked\n" @@ -253,10 +254,11 @@ def build_terminated_model_project_files() -> dict[str, str]: 'database = "terminated.duckdb"\n\n[settings]\nsql_analysis = true\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/m.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model m.', materialized table);\n\n" 'WITH a AS (SELECT id FROM __source("raw_orders"))\n' "SELECT id FROM a;\n" ), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 95931ffd37..07151eff97 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -33,7 +33,7 @@ def build_sqlbuild_project_with_manifest( sql: str for model_name, sql in model_sql_by_name.items(): project_dir.joinpath(f"models/{model_name}.sql").write_text( - f"MODEL ();\n\n{sql}\n", encoding="utf-8" + f"MODEL (description 'Test model.');\n\n{sql}\n", encoding="utf-8" ) project_dir.joinpath("target/manifest.json").write_text( manifest_source.read_text(encoding="utf-8"), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index fb053289c5..ed7eef6862 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -26,7 +26,7 @@ [ RealDbtManifestCompileTestCase( description="validates and preserves SQLBuild dbt ref from real dbt manifest", - sqlbuild_model_sql='MODEL ();\n\nselect order_id from __dbt_ref("stg_orders")\n', + sqlbuild_model_sql='MODEL (description "Test model.");\n\nselect order_id from __dbt_ref("stg_orders")\n', expected_compiled_sql='select order_id from __dbt_ref("stg_orders")', ), ], diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py index ba7d7f2900..f161626d30 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py @@ -25,7 +25,7 @@ description="plans dbt-only tag selector and skips SQLBuild work", args=("--select", "tag:nightly"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=(), expected_sqlbuild_command_argvs=(), @@ -42,7 +42,7 @@ description="plans mixed dbt and SQLBuild tag selector matches", args=("--select", "tag:nightly"), sqlbuild_model_contents_by_relative_path={ - "tagged_orders.sql": "MODEL (tags [nightly]);\n\nselect 1 as order_id\n", + "tagged_orders.sql": "MODEL (description 'Test model tagged_orders.', tags [nightly]);\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("tagged_orders",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "tagged_orders"),), @@ -60,10 +60,10 @@ args=("--select", "tag:nightly+"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders", "mart_orders"), @@ -104,7 +104,7 @@ description="plans SQLBuild-only tag selector and skips dbt work", args=("--select", "tag:sqb_only"), sqlbuild_model_contents_by_relative_path={ - "sqb_only.sql": "MODEL (tags [sqb_only]);\n\nselect 1 as order_id\n", + "sqb_only.sql": "MODEL (description 'Test model sqb_only.', tags [sqb_only]);\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("sqb_only",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "sqb_only"),), @@ -121,9 +121,9 @@ description="applies SQLBuild tag excludes after selection", args=("--select", "tag:sqb_only", "--exclude", "tag:deprecated"), sqlbuild_model_contents_by_relative_path={ - "sqb_only.sql": "MODEL (tags [sqb_only]);\n\nselect 1 as order_id\n", + "sqb_only.sql": "MODEL (description 'Test model sqb_only.', tags [sqb_only]);\n\nselect 1 as order_id\n", "deprecated_orders.sql": ( - "MODEL (tags [sqb_only, deprecated]);\n\nselect 2 as order_id\n" + "MODEL (description 'Test model deprecated_orders.', tags [sqb_only, deprecated]);\n\nselect 2 as order_id\n" ), }, expected_sqlbuild_model_names=("sqb_only",), @@ -141,7 +141,7 @@ description="plans SQLBuild-only model and skips dbt command work", args=("--select", "local_only"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("local_only",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "local_only"),), @@ -159,10 +159,10 @@ args=("--select", "+downstream_orders"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders",), @@ -197,10 +197,10 @@ args=("--select", "fact_orders+"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders", "mart_orders"), @@ -223,10 +223,10 @@ args=("--select", "fact_orders+", "--exclude", "mart_orders"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders",), @@ -246,7 +246,7 @@ description="routes explicit model path selector for real SQLBuild project", args=("--select", "path:models/marts"), sqlbuild_model_contents_by_relative_path={ - "marts/mart_orders.sql": "MODEL ();\n\nselect 1 as order_id\n", + "marts/mart_orders.sql": "MODEL (description 'Test model mart_orders.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("mart_orders",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "mart_orders"),), @@ -264,7 +264,7 @@ description="preserves routed SQLBuild cursor args in real plan argv", args=("--select", "local_only", "--start-cursor-int", "10"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("local_only",), expected_sqlbuild_command_argvs=( @@ -283,7 +283,7 @@ description="returns stable no-work plan for unmatched selectors", args=("--select", "does_not_exist"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=(), expected_sqlbuild_command_argvs=(), diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index 2e774c0b65..7d8cb469e7 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -752,9 +752,9 @@ class SyntheticSnowflakeProject: incremental_model_names: tuple[str, ...] -_TABLE_MODEL_HEADER: str = "MODEL (\n materialized table,\n);\n\n" +_TABLE_MODEL_HEADER: str = "MODEL (description 'Test model.',\n materialized table,\n);\n\n" _INCREMENTAL_MODEL_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor ordered_at,\n" @@ -953,6 +953,7 @@ def _sources_yml(*, database: str, source_schema: str, sources: int) -> str: lines.extend( ( f" - name: raw_orders_{index}", + f" description: Test source raw_orders_{index}.", f" database: {database.lower()}", f" schema: {source_schema}", f" table: orders_{index}", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py index fea5b2ad34..5d8df636e4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py @@ -68,7 +68,7 @@ def test_given_referenced_environment_change_when_fingerprinting_then_invalidate description="SQL output context preserves structural cache availability", relative_path="models/orders.sql", config=( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" "SELECT '${CTX:run_id}' AS invocation_id, 1 AS order_id\n" ), expected_available=True, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py index 0726159e65..74dab704e9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py @@ -55,7 +55,7 @@ def build_project_with_scenarios(project_dir: Path) -> CompiledProject: scenario_file=DiscoveredSqlScenarioFile( file_path=project_dir / relative_path, relative_path=Path(relative_path), - contents="SCENARIO();\n\nSELECT 1\n", + contents="SCENARIO(description 'Test scenario.');\n\nSELECT 1\n", header_values={}, sql_body="SELECT 1", name=scenario_name, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 5434cc973c..0ae7db5081 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -93,7 +93,7 @@ def prepare_static_compile_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) return project_dir @@ -533,18 +533,21 @@ def prepare_rule_gated_compile_project(root: Path, *, test_header: str) -> Path: (project_dir / "sources").mkdir() (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_refunds\n schema: main\n table: raw_refunds\n", + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_refunds\n description: Test source raw_refunds.\n" + " schema: main\n table: raw_refunds\n", encoding="utf-8", ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n', encoding="utf-8", ) (project_dir / "models" / "bad_star.sql").write_text( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n', + 'MODEL (description "Test model bad_star.", ' + 'materialized view);\n\nSELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) (project_dir / "tests" / "unit").mkdir(parents=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 252ef119be..dd8663809e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -280,7 +280,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " Wrote: target/compiled/", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " order_id (),\n" @@ -305,7 +305,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " = help: change the declared type or cast the expression explicitly", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " amount_cents (type INTEGER),\n" @@ -327,7 +327,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " = help: add the column to MODEL(columns) or remove it from the SELECT list", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " contract enforced,\n" " columns (\n" @@ -385,7 +385,7 @@ def test_given_contract_errors_when_running_compile_then_reports_diagnostics( expected_line=5, expected_column=5, model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " order_id (),\n" @@ -468,7 +468,7 @@ def test_given_explicit_column_contract_mode_when_compiling_column_audit_then_au encoding="utf-8", ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized view,\n" " columns (\n" " customer_id (audits [not_null]),\n" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py index a8aa26f04d..144dcae048 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py @@ -48,7 +48,7 @@ def prepare_static_dag_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) return project_dir @@ -67,7 +67,8 @@ def prepare_advisory_placement_dag_project(root: Path) -> Path: constants_dir.mkdir() (constants_dir / "value.sql").write_text("CONSTANT (name value, value 1);\n", encoding="utf-8") (project_dir / "models" / "orders.sql").write_text( - 'MODEL (materialized table);\n\nSELECT @const("value") AS order_id\n', + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT @const("value") AS order_id\n', encoding="utf-8", ) return project_dir @@ -98,6 +99,7 @@ def prepare_python_dag_project(root: Path) -> Path: " meta={'owner': 'data'},", ")", "def prepare_orders(ctx):", + " '''Test task prepare_orders.'''", " return ctx.result(payload={'rows': 1}, metadata={'source': 'fixture'})", ) ) @@ -115,6 +117,7 @@ def prepare_python_dag_project(root: Path) -> Path: " columns=[{'name': 'order_id', 'type': 'integer'}],", ")", "def warehouse_export(ctx):", + " '''Test loader warehouse_export.'''", " return [{'order_id': 1}]", ) ) @@ -176,6 +179,7 @@ def prepare_python_dag_project(root: Path) -> Path: "", "@check(depends_on=warehouse_export, tags=['loader'])", "def check_loader_export(ctx):", + " '''Test check check_loader_export.'''", " return ctx.pass_(message='loader rows available')", ) ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 7d34289ba8..a062c17353 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -154,6 +154,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit "kind": "python_check", "name": "check_loader_export", "checked_asset_ids": ["loader:warehouse_export"], + "description": "Test check check_loader_export.", "path": "python/checks/check_loader_export.py", "severity": "error", "tags": ["loader"], diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index d55f1ec672..18a979f4fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -306,7 +306,7 @@ def singular_audit_files(*, base: dict[str, str], sql: str) -> dict[str, str]: def model_header(*, key: str, value: str) -> str: """Return a one-column model whose header sets one key.""" - return f"MODEL ({key} {value});\nSELECT 1 AS order_id" + return f"MODEL (description 'Test model check.', {key} {value});\nSELECT 1 AS order_id" def compile_and_assemble(*, project_dir: Path) -> CompiledProject: @@ -323,7 +323,7 @@ def gate_audit(*, read: str) -> str: return f"AUDIT ();\nSELECT s.* FROM @relation s LEFT JOIN {read} a USING (code)" -def gate_model(*, sql: str, header: str = "MODEL ();") -> str: +def gate_model(*, sql: str, header: str = "MODEL (description 'Test model.');") -> str: """Return a model with the given header.""" return f"{header}\n{sql}" @@ -343,7 +343,7 @@ def execution_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, st def inline_sql_hook_header(sql: str) -> str: """Return a MODEL header running one inline SQL pre-hook.""" - return f"MODEL (pre_hooks [inline_sql('{sql}')]);" + return f"MODEL (description 'Test model.', pre_hooks [inline_sql('{sql}')]);" def lineage_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, str]]: @@ -447,7 +447,7 @@ def python_loader_source(*, depends_on: str, body: str) -> str: "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" f"@loader(depends_on=[{depends_on}])\n" "def raw_customers(ctx):\n" f"{body}" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index c5d8619aaa..30ab6049fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -53,6 +53,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. contract: enforced expression: "(SELECT 1 AS order_id)" columns: @@ -62,7 +63,7 @@ """.strip() + "\n", "models/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT order_id FROM __source("raw_orders") """.strip() @@ -70,10 +71,10 @@ } _SELECTION_REPO_FILES: dict[str, str] = { "sqlbuild_project.toml": 'name = "selection_demo"\nadapter = "duckdb"\n', - "models/root.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "models/middle.sql": 'MODEL ();\n\nSELECT id FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT id FROM __ref("middle")\n', - "models/unrelated.sql": "MODEL ();\n\nSELECT 2 AS id\n", + "models/root.sql": "MODEL (description 'Test model root.');\n\nSELECT 1 AS id\n", + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT id FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT id FROM __ref("middle")\n', + "models/unrelated.sql": "MODEL (description 'Test model unrelated.');\n\nSELECT 2 AS id\n", } @@ -229,7 +230,7 @@ def test_given_one_changed_model_when_entry_rows_are_absent_then_compact_batch_r with sqlite3.connect(cache_path) as connection: _ = connection.execute("DELETE FROM model_analysis") (tmp_path / "models" / "unrelated.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", + "MODEL (description 'Test model unrelated.');\n\nSELECT 3 AS id\n", encoding="utf-8", ) @@ -290,7 +291,7 @@ def test_given_changed_expanded_sql_when_compiling_then_writes_a_new_analysis_ob ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - changed_sql = 'MODEL ();\n\nSELECT order_id + 1 AS order_id FROM __source("raw_orders")\n' + changed_sql = 'MODEL (description "Test model.");\n\nSELECT order_id + 1 AS order_id FROM __source("raw_orders")\n' (tmp_path / "models" / "orders.sql").write_text(changed_sql, encoding="utf-8") analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) @@ -766,7 +767,7 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( tmp_path, { **_SELECTION_REPO_FILES, - "models/root.sql": "MODEL ();\n\nSELECT CAST(NULL AS INTEGER) AS id\n", + "models/root.sql": "MODEL (description 'Test model root.');\n\nSELECT CAST(NULL AS INTEGER) AS id\n", }, ) _ = compile_project_with_cache(project_dir=tmp_path) @@ -775,8 +776,8 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( unrelated_path: Path = tmp_path / "models" / "unrelated.sql" unrelated_path.write_text( unrelated_path.read_text(encoding="utf-8").replace( - "MODEL ();", - "MODEL (columns (id (nullable false)));", + "MODEL (description 'Test model unrelated.');", + "MODEL (description 'Test model unrelated.', columns (id (nullable false)));", 1, ), encoding="utf-8", @@ -788,8 +789,8 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( root_path: Path = tmp_path / "models" / "root.sql" root_path.write_text( root_path.read_text(encoding="utf-8").replace( - "MODEL ();", - "MODEL (columns (id (nullable false)));", + "MODEL (description 'Test model root.');", + "MODEL (description 'Test model root.', columns (id (nullable false)));", 1, ), encoding="utf-8", @@ -818,7 +819,7 @@ def test_given_model_sql_change_with_stable_signature_when_compiling_then_downst analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", + "MODEL (description 'Test model root.');\n\nSELECT 3 AS id\n", encoding="utf-8", ) @@ -843,7 +844,7 @@ def test_given_model_output_change_when_compiling_then_downstream_closure_misses analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) @@ -869,7 +870,9 @@ def test_given_changed_output_signature_when_reanalyzing_downstream_then_uses_de _ = compile_project_with_cache(project_dir=tmp_path) root_path: Path = tmp_path / "models" / "root.sql" original_sql: str = root_path.read_text(encoding="utf-8") - root_path.write_text("MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8") + root_path.write_text( + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8" + ) _ = compile_project_with_cache(project_dir=tmp_path) batcher: Mock = Mock(wraps=assembly_project.analyze_queries_with_compact_polyglot_batch) monkeypatch.setattr(assembly_project, "analyze_queries_with_compact_polyglot_batch", batcher) @@ -896,7 +899,9 @@ def test_given_cached_model_signature_is_restored_when_compiling_then_downstream _ = compile_project_with_cache(project_dir=tmp_path) root_path: Path = tmp_path / "models" / "root.sql" original_sql: str = root_path.read_text(encoding="utf-8") - root_path.write_text("MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8") + root_path.write_text( + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8" + ) _ = compile_project_with_cache(project_dir=tmp_path) analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) @@ -922,13 +927,13 @@ def test_given_selected_upstream_change_when_compiling_full_project_then_stale_c tmp_path, { **_SELECTION_REPO_FILES, - "models/middle.sql": 'MODEL ();\n\nSELECT * FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT * FROM __ref("middle")\n', + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT * FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT * FROM __ref("middle")\n', }, ) _ = compile_project_with_cache(project_dir=tmp_path) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) _ = compile_project_with_cache( @@ -958,14 +963,14 @@ def test_given_upstream_interface_change_when_compact_batch_exists_then_dependen tmp_path, { **_SELECTION_REPO_FILES, - "models/middle.sql": 'MODEL ();\n\nSELECT * FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT * FROM __ref("middle")\n', + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT * FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT * FROM __ref("middle")\n', }, ) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) _ = compile_project_with_cache(project_dir=tmp_path) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) @@ -1060,7 +1065,7 @@ def test_given_unselected_invalid_reference_when_compiling_then_live_validation_ write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: repo_files: dict[str, str] = _SELECTION_REPO_FILES | { - "models/unrelated.sql": 'MODEL ();\n\nSELECT id FROM __ref("missing")\n' + "models/unrelated.sql": 'MODEL (description "Test model unrelated.");\n\nSELECT id FROM __ref("missing")\n' } write_repo_files(tmp_path, repo_files) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 86538d67c4..d0e4494d48 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -58,7 +58,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (type VARCHAR), ), @@ -71,6 +71,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: public table: orders """.strip() @@ -78,6 +79,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: code type: VARCHAR @@ -132,10 +134,11 @@ repo_files=base_repo_files() | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", "seeds/schema.yml": """ seeds: - name: order_codes + description: Test seed order_codes. columns: - name: order_id type: INTEGER @@ -209,6 +212,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: code type: VARCHAR @@ -262,6 +266,7 @@ "seeds/lookups.yml": """ seeds: - name: country_codes + description: Test seed country_codes. database: "${if(ENV:USE_SEED_DB, 'seed_db', CTX:destination.database)}" schema: "${CTX:destination.schema}_${seed_schema_suffix}" columns: @@ -300,9 +305,9 @@ 'def normalize_status(value):\n return f"LOWER(TRIM({value}))"\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" + "MODEL (description 'Test model orders.');\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" ), - "models/customers.sql": "MODEL ();\n\nSELECT 'active' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 'active' AS status\n", "tests/unit/test_normalize_status.sql": """ TEST (mode macro, name "normalizes_status"); @@ -354,6 +359,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -363,6 +369,7 @@ @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"id": 1}] """.strip() + "\n", @@ -387,7 +394,7 @@ def raw_orders(ctx): | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/format_cents.sql": """ -FUNCTION ( +FUNCTION (description "Test function format_cents.", arguments (amount_cents INTEGER), returns VARCHAR, ); @@ -396,9 +403,9 @@ def raw_orders(ctx): """.strip() + "\n", "models/orders.sql": ( - 'MODEL ();\n\nSELECT __udf("format_cents")(1250) AS formatted\n' + 'MODEL (description "Test model orders.");\n\nSELECT __udf("format_cents")(1250) AS formatted\n' ), - "models/customers.sql": "MODEL ();\n\nSELECT 'active' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 'active' AS status\n", "tests/unit/test_format_cents.sql": """ TEST (mode udf, name "formats_cents"); @@ -438,7 +445,7 @@ def raw_orders(ctx): | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( customer_id INTEGER, @@ -450,7 +457,7 @@ def raw_orders(ctx): """.strip() + "\n", "models/customer_order_summary.sql": ( - "MODEL ();\n\nSELECT customer_id, order_id " + "MODEL (description 'Test model customer_order_summary.');\n\nSELECT customer_id, order_id " 'FROM __table_fn("customer_orders")(42)\n' ), "tests/unit/test_customer_orders.sql": """ @@ -593,7 +600,7 @@ def test_given_named_generic_audit_instances_when_assembling_then_each_name_is_p base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [ expression_is_true (name "positive_revenue", expression "revenue > 0"), expression_is_true (name "bounded_revenue", expression "revenue < 1000"), @@ -666,12 +673,14 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. database: declared_seed_db schema: declared_seed_schema columns: - name: code type: VARCHAR - name: currencies + description: Test seed currencies. columns: - name: code type: VARCHAR @@ -802,12 +811,12 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t AssembleSqlHookValidationTestCase( description="validates named and inline payloads with adapter dialect", model_sql=( - 'MODEL (pre_hooks [sql("vendor_hook")], ' + 'MODEL (description "Test model.", pre_hooks [sql("vendor_hook")], ' 'post_hooks [inline_sql("SELECT [other] FROM [orders]; ' 'SELECT [other] FROM [orders]")]);\n\nSELECT 1 AS id\n' ), named_hook_sql=( - "HOOK ();\n\nSELECT [value] FROM [orders]; SELECT [value] FROM [orders]\n" + "HOOK (description 'Test hook.');\n\nSELECT [value] FROM [orders]; SELECT [value] FROM [orders]\n" ), dialect="tsql", expected_hook_statements=( @@ -818,11 +827,11 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t AssembleSqlHookValidationTestCase( description="skips named and inline Polyglot validation for model opt out", model_sql=( - 'MODEL (sql_validation false, pre_hooks [sql("vendor_hook")], ' + 'MODEL (description "Test model.", sql_validation false, pre_hooks [sql("vendor_hook")], ' 'post_hooks [inline_sql("DBCC CHECKIDENT (orders, RESEED, 0)")]);' "\n\nSELECT 1 AS id\n" ), - named_hook_sql="HOOK ();\n\nDBCC CHECKIDENT ('orders', RESEED, 0)\n", + named_hook_sql="HOOK (description 'Test hook.');\n\nDBCC CHECKIDENT ('orders', RESEED, 0)\n", dialect="tsql", expected_hook_statements=( "DBCC CHECKIDENT ('orders', RESEED, 0)", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py index 861e1d6b35..ded2583fcb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py @@ -28,18 +28,14 @@ sql_analysis = false sql_validation = false """ -_SOURCES_WITH_AUDIT: str = ( - "sources:\n - name: raw_orders\n table: orders\n audits:\n - code_check\n" -) +_SOURCES_WITH_AUDIT: str = "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n audits:\n - code_check\n" _SEED_WITH_AUDIT: str = ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n columns:\n - name: code\n type: VARCHAR\n" " audits:\n - code_check\n" ) -_AUDITED_HEADER: str = "MODEL (audits [code_check]);" -_ORDER_CHECK_HEADER: str = "MODEL (audits [order_check]);" -_ALLOWED_SEED: str = ( - "seeds:\n - name: allowed_codes\n columns:\n - name: code\n type: VARCHAR\n" -) +_AUDITED_HEADER: str = "MODEL (description 'Test model.', audits [code_check]);" +_ORDER_CHECK_HEADER: str = "MODEL (description 'Test model.', audits [order_check]);" +_ALLOWED_SEED: str = "seeds:\n - name: allowed_codes\n description: Test seed allowed_codes.\n columns:\n - name: code\n type: VARCHAR\n" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 88a38806c0..4f1801971c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -45,7 +45,7 @@ def test_given_factory_attachment_when_building_inputs_then_cases_append_with_pr tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", + "models/orders.sql": f"MODEL (description 'Test model orders.', {test_case.model_header}); SELECT 1 AS amount", "audits/generic/expression_is_true.sql": _AUDIT_FILE, "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @@ -113,7 +113,7 @@ def test_given_invalid_factory_attachment_when_building_inputs_then_compile_erro tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", + "models/orders.sql": f"MODEL (description 'Test model orders.', {test_case.model_header}); SELECT 1 AS amount", "python/factories/quality.py": f""" from sqlbuild.audits import AuditCase, audit_factory diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 018008170d..922a47a23a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -134,14 +134,14 @@ def test_given_visibility_record_when_rebinding_then_projects_new_consumer_indep "global declaration", {"constants/value.sql": "CONSTANT (name value, value 1);"}, "models/domain/child/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 1 AS value", ), ScopedDeclarationCompileTestCase( "inherited declaration", {"models/domain/constants/value.sql": "CONSTANT (name value, value 2);"}, "models/domain/child/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 2 AS value", ), ScopedDeclarationCompileTestCase( @@ -153,21 +153,23 @@ def test_given_visibility_record_when_rebinding_then_projects_new_consumer_indep ), }, "models/domain/child/grandchild/orders.sql", - ('MODEL ();\nSELECT @const("parent_value") + @const("child_value") AS value\n'), + ( + 'MODEL (description "Test model orders.");\nSELECT @const("parent_value") + @const("child_value") AS value\n' + ), "SELECT 2 + 3 AS value", ), ScopedDeclarationCompileTestCase( "local declaration", {"models/domain/_constants/value.sql": "CONSTANT (name value, value 4);"}, "models/domain/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 4 AS value", ), ScopedDeclarationCompileTestCase( "private declaration", {}, "models/domain/orders.sql", - 'MODEL (constants (_value 5));\nSELECT @const("_value") AS value\n', + 'MODEL (description "Test model orders.", constants (_value 5));\nSELECT @const("_value") AS value\n', "SELECT 5 AS value", ), ), @@ -198,7 +200,7 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "sibling declaration", { "models/one/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/two/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/two/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ("known but inaccessible", "models/one/_constants/value.sql", "models/two/orders.sql"), ), @@ -206,19 +208,21 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "descendant declaration", { "models/domain/child/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ("known but inaccessible", "models/domain/child/_constants/value.sql"), ), ScopedDeclarationErrorTestCase( "unknown declaration", - {"models/orders.sql": 'MODEL ();\nSELECT @const("missing") AS value'}, + { + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("missing") AS value' + }, ("Unknown constant 'missing'", "Visible constants: none"), ), ScopedDeclarationErrorTestCase( "model private does not export to test", { - "models/orders.sql": "MODEL (constants (_value 1));\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.', constants (_value 1));\nSELECT 1 AS value", "tests/unit/orders.sql": ( 'TEST ();\nWITH __expected__orders AS (SELECT @const("_value") AS value) ' "SELECT 1" @@ -230,7 +234,7 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "filename resemblance without expected relationship", { "models/domain/_constants/value.sql": "CONSTANT (name model_value, value 9);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/orders.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__assert__valid AS (SELECT @const('model_value') WHERE FALSE) SELECT 1" @@ -241,11 +245,11 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib ScopedDeclarationErrorTestCase( "unrelated production scope is not granted", { - "models/orders/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "models/customers/_constants/value.sql": ( "CONSTANT (name customer_value, value 10);" ), - "models/customers/customers.sql": "MODEL ();\nSELECT 1 AS value", + "models/customers/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('customer_value') AS value) SELECT 1" @@ -278,7 +282,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti { "models/domain/_constants/value.sql": "CONSTANT (name value, value 11);", "models/domain/orders.sql": ( - "MODEL (pre_hooks [inline_sql(\"SELECT @const('value')\")]);\nSELECT 1" + "MODEL (description 'Test model orders.', pre_hooks [inline_sql(\"SELECT @const('value')\")]);\nSELECT 1" ), }, lambda inputs: ( @@ -292,8 +296,8 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "named hook definition path", { "models/domain/_sqlbuild/_constants/value.sql": "CONSTANT (name value, value 12);", - "models/domain/_sqlbuild/_hooks/sql/typed.sql": 'HOOK ();\nSELECT @const("value")', - "models/domain/orders.sql": 'MODEL (post_hooks [sql("typed")]);\nSELECT 1', + "models/domain/_sqlbuild/_hooks/sql/typed.sql": 'HOOK (description "Test hook typed.");\nSELECT @const("value")', + "models/domain/orders.sql": 'MODEL (description "Test model orders.", post_hooks [sql("typed")]);\nSELECT 1', }, lambda inputs: ( cast(list[SqlHookEntry], inputs.model_inputs[0].config.values["post_hooks"])[ @@ -306,8 +310,10 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "function definition path", { "functions/sql/domain/_constants/value.sql": "CONSTANT (name value, value 13);", - "functions/sql/domain/value.sql": ('FUNCTION (returns INTEGER);\n@const("value")'), - "models/orders.sql": "MODEL ();\nSELECT 1", + "functions/sql/domain/value.sql": ( + 'FUNCTION (description "Test function value.", returns INTEGER);\n@const("value")' + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, lambda inputs: inputs.sql_function_inputs[0].body_sql, "13", @@ -320,8 +326,8 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti 'AUDIT ();\nSELECT @const("value") FROM __ref("orders") ' 'JOIN __ref("customers") ON TRUE' ), - "models/domain/orders.sql": "MODEL ();\nSELECT 1", - "models/domain/customers.sql": "MODEL ();\nSELECT 1", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", + "models/domain/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1", }, lambda inputs: inputs.audit_inputs[0].sql_body, 'SELECT 15 FROM __ref("orders") JOIN __ref("customers") ON TRUE', @@ -335,7 +341,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "models/domain/_sqlbuild/_audits/generic/scoped.sql": ( 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE @const("value") = 18' ), - "models/domain/orders.sql": "MODEL (audits [scoped]);\nSELECT 1", + "models/domain/orders.sql": "MODEL (description 'Test model orders.', audits [scoped]);\nSELECT 1", }, lambda inputs: inputs.audit_inputs[0].sql_body, 'SELECT * FROM __ref("orders") WHERE 18 = 18', @@ -348,7 +354,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " '__expected__orders AS (SELECT @const("value") AS value) SELECT 1' ), - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, lambda inputs: inputs.test_inputs[0].sql_body, "WITH __ref__orders AS (SELECT 1 AS value), " @@ -359,10 +365,10 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti { "tests/scenarios/domain/_constants/value.sql": ("CONSTANT (name value, value 17);"), "tests/scenarios/domain/orders.sql": ( - "SCENARIO ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "SCENARIO (description 'Test scenario orders.');\nWITH __ref__orders AS (SELECT 1 AS value), " '__expected__orders AS (SELECT @const("value") AS value) SELECT 1' ), - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, lambda inputs: inputs.scenario_inputs[0].sql_body, "WITH __ref__orders AS (SELECT 1 AS value), " @@ -395,7 +401,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ ), "models/domain/_constants/local.sql": ("CONSTANT (name local_value, value 3);"), "models/domain/_enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value, 'OPEN' AS state", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value, 'OPEN' AS state", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('inherited_value') + " @@ -413,9 +419,9 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="multiple expected models deterministic union", files={ "models/a/_constants/a.sql": "CONSTANT (name a_value, value 4);", - "models/a/a.sql": "MODEL ();\nSELECT 4 AS value", + "models/a/a.sql": "MODEL (description 'Test model a.');\nSELECT 4 AS value", "models/b/_constants/b.sql": "CONSTANT (name b_value, value 5);", - "models/b/b.sql": "MODEL ();\nSELECT 5 AS value", + "models/b/b.sql": "MODEL (description 'Test model b.');\nSELECT 5 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__a AS (SELECT 1 AS value), " "__expected__b AS (SELECT @const('b_value') AS value), " @@ -432,7 +438,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="test path visibility remains separate from model grant", files={ "models/domain/_constants/model.sql": "CONSTANT (name model_value, value 6);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/_constants/test.sql": "CONSTANT (name test_value, value 7);", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -453,7 +459,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' ), "models/domain/orders.sql": ( - "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + "MODEL (description 'Test model orders.');\nSELECT @discount('10') AS value, @surcharge('10') AS fee" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -473,7 +479,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' ), "models/domain/orders.sql": ( - "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + "MODEL (description 'Test model orders.');\nSELECT @discount('10') AS value, @surcharge('10') AS fee" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -490,7 +496,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="model private declaration is not granted", files={ "models/orders.sql": ( - "MODEL (constants (_private 8));\nSELECT @const('_private') AS value" + "MODEL (description 'Test model orders.', constants (_private 8));\nSELECT @const('_private') AS value" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -503,7 +509,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ ExpectedModelDeclarationGrantTestCase( description="filename resemblance and no expected model grant nothing", files={ - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/orders.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__assert__valid AS (SELECT 1 WHERE FALSE) SELECT 1" @@ -516,9 +522,9 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="scenario expected model grant", files={ "models/domain/_constants/value.sql": "CONSTANT (name scenario_value, value 10);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/scenarios/check.sql": ( - "SCENARIO ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "SCENARIO (description 'Test scenario check.');\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('scenario_value') AS value) SELECT 1" ), }, @@ -568,7 +574,7 @@ def test_given_expected_models_when_compiling_then_public_declarations_are_grant description="expected model provenance is retained", files={ "models/domain/_constants/value.sql": ("CONSTANT (name model_value, value 6);"), - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS " @@ -608,15 +614,15 @@ def test_given_relationship_granted_reference_when_compiling_then_usage_retains_ description="exact local placement is accepted", files={ "models/domain/_constants/value.sql": ("CONSTANT (name value, value 1);"), - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ), ScopePlacementCompileTestCase( description="inherited lowest common ancestor is accepted", files={ "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/a/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', - "models/domain/b/customers.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/a/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', + "models/domain/b/customers.sql": 'MODEL (description "Test model customers.");\nSELECT @const("value") AS value', }, ), ), @@ -660,8 +666,8 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu { "sqlbuild_project.toml": _PROJECT_FILE, "models/domain/area/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/area/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', - "models/domain/area/customers.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/area/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', + "models/domain/area/customers.sql": 'MODEL (description "Test model customers.");\nSELECT @const("value") AS value', }, ) @@ -681,7 +687,7 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="used global with one owner is rejected as over broad", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_fragment="required exact-owner-private at 'models'", ), @@ -689,14 +695,16 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="unused global is rejected", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, expected_fragment="Unused global declaration 'constant:value'", ), ScopePlacementCompileTestCase( description="unused private is rejected", files={ - "models/orders.sql": ("MODEL (constants (_value 1));\nSELECT 1 AS value"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', constants (_value 1));\nSELECT 1 AS value" + ), }, expected_fragment="Unused private declaration 'constant:model:orders._value'", ), @@ -704,7 +712,7 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="inherited declaration used in one directory must be local", files={ "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_fragment="required exact-owner-private at 'models/domain'", ), @@ -737,7 +745,7 @@ def test_given_invalid_declaration_placement_when_assembling_then_it_reports_an_ description="over broad global is advisory when placement enforcement is disabled", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_model_names=("orders",), expected_diagnostics=(("S024", "warning"),), @@ -777,7 +785,7 @@ def test_given_disabled_placement_enforcement_when_assembling_then_project_and_w files={"models/domain/enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);"}, model_path="models/domain/child/orders.sql", model_sql=( - "MODEL (contract enforced, columns (state (type state)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (state (type state)));\n" 'SELECT @enum("state").OPEN AS state' ), expected_sql="SELECT 'OPEN' AS state", @@ -814,7 +822,7 @@ def test_given_inherited_enum_when_compiling_contract_then_resolves_from_model_p description="source authored path", files={ "sources/domain/_constants/value.sql": "CONSTANT (name value, value 14);", - "models/orders.sql": "MODEL ();\nSELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, model_path="sources/domain/source.yml", model_sql='SELECT @const("value")', @@ -874,7 +882,7 @@ def test_given_sources_in_same_file_when_reusing_visibility_then_usage_consumers { "sqlbuild_project.toml": _PROJECT_FILE, "sources/domain/_constants/value.sql": "CONSTANT (name value, value 14);", - "models/orders.sql": "MODEL ();\nSELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) @@ -932,7 +940,7 @@ def test_given_tests_in_same_directory_when_reusing_visibility_then_usage_consum tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/domain/_constants/value.sql": "CONSTANT (name value, value 16);", "tests/unit/domain/customer_orders.sql": test_sql, "tests/unit/domain/inventory_orders.sql": test_sql, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index f85ca4e113..33c0796275 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -81,7 +81,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop CONSTANT (name source_name, value "O'Brien"); """, "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", contract enforced, pre_hooks [inline_sql('SELECT @const("min_items") + @const("_offset")')], enums ( @@ -107,7 +107,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop @const("source_name") AS source_name """, "functions/sql/add_threshold.sql": """ -FUNCTION ( +FUNCTION (description "Test function add_threshold.", arguments (value INTEGER), returns INTEGER, ); @@ -117,6 +117,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop "sources/inline.yml": """ sources: - name: inline_values + description: Test source inline_values. expression: | SELECT @const("min_items") AS value """, @@ -132,7 +133,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop SELECT 1 """, "tests/scenarios/orders.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario orders."); WITH __ref__orders AS ( SELECT @const("min_items") AS threshold @@ -321,7 +322,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ CONSTANT (name labels, value (GB "Great Britain", FR "France")); """, "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [inline_sql("SELECT 'GB' IN @const('countries')")], post_hooks [sql("typed_hook")], audits [typed_audit], @@ -338,7 +339,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ @const("_local_values") AS local_values """, "functions/sql/is_supported.sql": """ -FUNCTION (arguments (value VARCHAR), returns BOOLEAN); +FUNCTION (description "Test function is_supported.", arguments (value VARCHAR), returns BOOLEAN); value IN @const("countries") """, "hooks/sql/typed_hook.sql": """ @@ -348,6 +349,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ "sources/inline.yml": """ sources: - name: supported_entries + description: Test source supported_entries. expression: | SELECT * FROM entries WHERE country IN @const("countries") """, @@ -358,7 +360,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ SELECT 1 """, "tests/scenarios/orders.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario orders."); WITH __ref__orders AS (SELECT @const("country_array") AS countries), __expected__orders AS (SELECT @const("country_array") AS countries) SELECT 1 @@ -405,7 +407,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/threshold.sql", initial_declaration="CONSTANT (name threshold, value 7);\n", changed_declaration="CONSTANT (name threshold, value 8);\n", - model_sql='MODEL ();\nSELECT @const("threshold") AS threshold\n', + model_sql='MODEL (description "Test model threshold.");\nSELECT @const("threshold") AS threshold\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -414,7 +416,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["FR", "GB"]);\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -423,7 +425,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["GB", "FR", "FR"]);\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -432,7 +434,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value {"GB", "FR"});\n', changed_declaration='CONSTANT (name countries, value {"FR", "GB"});\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=False, expected_metadata_changed=False, ), @@ -445,7 +447,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ changed_declaration=( 'CONSTANT (name labels, value (FR "France", GB "Great Britain"));\n' ), - model_sql='MODEL ();\nSELECT @const("labels") AS labels\n', + model_sql='MODEL (description "Test model.");\nSELECT @const("labels") AS labels\n', expected_query_hash_changed=False, expected_metadata_changed=False, ), @@ -454,7 +456,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value {"GB", "FR"});\n', changed_declaration='CONSTANT (name countries, value {"GB", "HK"});\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -463,7 +465,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/labels.sql", initial_declaration='CONSTANT (name labels, value (GB "Great Britain"));\n', changed_declaration='CONSTANT (name labels, value (GB "Britain"));\n', - model_sql='MODEL ();\nSELECT @const("labels") AS labels\n', + model_sql='MODEL (description "Test model labels.");\nSELECT @const("labels") AS labels\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -474,7 +476,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ changed_declaration=( 'CONSTANT (name countries, value ["GB", "FR"], render_as array);\n' ), - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -483,11 +485,13 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["GB", "HK"]);\n', - model_sql="MODEL ();\nSELECT 1 AS value\n", + model_sql="MODEL (description 'Test model countries.');\nSELECT 1 AS value\n", expected_query_hash_changed=False, expected_metadata_changed=False, additional_files={ - "models/z_consumer.sql": ('MODEL ();\nSELECT @const("countries") AS countries\n') + "models/z_consumer.sql": ( + 'MODEL (description "Test model z_consumer.");\nSELECT @const("countries") AS countries\n' + ) }, ), DeclarationFingerprintTestCase( @@ -496,7 +500,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ initial_declaration="ENUM (name state, members [OPEN]);\n", changed_declaration="ENUM (name state, members [OPEN, CLOSED]);\n", model_sql=""" -MODEL ( +MODEL (description "Test model state.", contract enforced, columns (state (type state)), ); @@ -547,7 +551,9 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="unknown enum member", repo_files={ "enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);", - "models/orders.sql": ('MODEL ();\nSELECT @enum("state").MISSING AS state\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\nSELECT @enum("state").MISSING AS state\n' + ), }, expected_error_fragment="Unknown member 'MISSING' for enum 'state'", ), @@ -555,29 +561,31 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="lowercase enum member access", repo_files={ "enums/state.sql": 'ENUM (name state, members (DELIVERY "active"));', - "models/orders.sql": ('MODEL ();\nSELECT @enum("state").active AS state\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\nSELECT @enum("state").active AS state\n' + ), }, expected_error_fragment="Unknown member 'active' for enum 'state'", ), CompileDeclarationsErrorTestCase( description="unknown constant", repo_files={ - "models/orders.sql": 'MODEL ();\nSELECT @const("missing") AS value\n', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("missing") AS value\n', }, expected_error_fragment="Unknown constant 'missing'", ), CompileDeclarationsErrorTestCase( description="foreign private constant", repo_files={ - "models/a.sql": "MODEL (constants (_limit 7));\nSELECT 1 AS value\n", - "models/b.sql": 'MODEL ();\nSELECT @const("_limit") AS value\n', + "models/a.sql": "MODEL (description 'Test model a.', constants (_limit 7));\nSELECT 1 AS value\n", + "models/b.sql": 'MODEL (description "Test model b.");\nSELECT @const("_limit") AS value\n', }, expected_error_fragment="Constant '_limit' is known but inaccessible", ), CompileDeclarationsErrorTestCase( description="non-private model constant name", repo_files={ - "models/orders.sql": "MODEL (constants (limit 7));\nSELECT 1 AS value\n", + "models/orders.sql": "MODEL (description 'Test model orders.', constants (limit 7));\nSELECT 1 AS value\n", }, expected_error_fragment=( "Invalid model-local constant identity 'limit'.*use snake_case '_limit'" @@ -588,7 +596,7 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity repo_files={ "enums/one.sql": "ENUM (name state, members [OPEN]);", "enums/nested/two.sql": "ENUM (name state, members [CLOSED]);", - "models/orders.sql": "MODEL ();\nSELECT 1 AS value\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value\n", }, expected_error_fragment="Duplicate declaration 'enum:state'", ), @@ -596,7 +604,7 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="nested collection cannot render as a portable value list", repo_files={ "constants/groups.sql": "CONSTANT (name groups, value [[1], [2]]);", - "models/orders.sql": 'MODEL ();\nSELECT @const("groups") AS groups\n', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("groups") AS groups\n', }, expected_error_fragment=( "constants/groups.sql constant 'groups'.*adapter 'duckdb'.*value_list" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py index 5e1598f99d..9e8b4fccbe 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py @@ -31,15 +31,19 @@ sql_validation = false """ _HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py" -_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);' -_COUNTRY_SEED: str = ( - "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n" +_HOOKED_HEADER: str = ( + 'MODEL (description "Test model lookups.", post_hooks [python("refresh_lookup")]);' +) +_COUNTRY_SEED: str = "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n - name: code\n type: VARCHAR\n" +_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n" ) -_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n" _SQL_HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/sql/record_rows.sql" _MACRO_PATH: str = "models/marts/_sqlbuild/_macros/counts.py" _IGNORING_MACRO: str = "def count_rows(relation):\n return 'SELECT 1'\n" -_NAMED_SQL_HOOK_HEADER: str = 'MODEL (post_hooks [sql("record_rows")]);' +_NAMED_SQL_HOOK_HEADER: str = ( + 'MODEL (description "Test model counts.", post_hooks [sql("record_rows")]);' +) @pytest.mark.parametrize( @@ -86,7 +90,7 @@ description="named SQL hook passing a model to a macro that ignores it", files={ _MACRO_PATH: _IGNORING_MACRO, - _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("customers"))\n', + _SQL_HOOK_PATH: 'HOOK (description "Test hook.");\n@count_rows(__ref("customers"))\n', "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"), "models/marts/orders.sql": gate_model( sql="SELECT 1 AS order_id", header=_NAMED_SQL_HOOK_HEADER @@ -199,11 +203,11 @@ def test_given_sql_hook_read_when_compiling_then_lineage_and_selection_ignore_it description="inline and named SQL hooks reading a model built from their model", files={ _MACRO_PATH: _IGNORING_MACRO, - _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("order_count"))\n', + _SQL_HOOK_PATH: 'HOOK (description "Test hook.");\n@count_rows(__ref("order_count"))\n', "models/marts/orders.sql": gate_model( sql="SELECT 1 AS order_id", header=( - "MODEL (pre_hooks [inline_sql('SELECT * FROM __ref(\"order_rollup\")')], " + "MODEL (description 'Test model orders.', pre_hooks [inline_sql('SELECT * FROM __ref(\"order_rollup\")')], " 'post_hooks [sql("record_rows")]);' ), ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 40e6fc1a11..71a1e83e07 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -60,7 +60,7 @@ def test_given_identical_contract_headers_when_attaching_then_each_model_keeps_i test_case: ExpectedCountTestCase, ) -> None: model_sql: str = """ -MODEL ( +MODEL (description "Test model.", columns (order_id (type INTEGER, audits [not_null])), ); SELECT 1::INTEGER AS order_id @@ -405,7 +405,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac { "sqlbuild_project.toml": _PROJECT_FILE, "models/stg_order_amounts.sql": """ -MODEL ( +MODEL (description "Test model stg_order_amounts.", contract enforced, columns ( customer_id (type INTEGER), @@ -419,7 +419,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac CAST(10.25 AS DECIMAL(12,2)) AS amount """, "models/customer_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -437,7 +437,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac GROUP BY customer_id """, "models/customer_category_amounts_copy.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts_copy.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -522,7 +522,7 @@ def test_given_partial_upstream_schema_when_compiling_dynamic_pivot_then_closure { "sqlbuild_project.toml": _PROJECT_FILE, "models/order_amounts.sql": """ -MODEL ( +MODEL (description "Test model order_amounts.", contract none, columns ( customer_id (type INTEGER), @@ -537,7 +537,7 @@ def test_given_partial_upstream_schema_when_compiling_dynamic_pivot_then_closure CAST(99 AS INTEGER) AS unrelated """, "models/customer_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -594,7 +594,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu { "sqlbuild_project.toml": _PROJECT_FILE, "models/order_amounts.sql": """ -MODEL ( +MODEL (description "Test model order_amounts.", contract enforced, columns ( customer_id (type INTEGER), @@ -610,7 +610,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu CAST(99 AS INTEGER) AS unrelated """, "models/open_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model open_category_amounts.", contract none, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -628,7 +628,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu GROUP BY customer_id """, "models/closed_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model closed_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -826,7 +826,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ "schemas/order.sql": """ SCHEMA (name order, extends missing, columns (id (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="extends unknown schema 'missing'", ), @@ -838,7 +838,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ SCHEMA (name second, extends third, columns (second_id (type INTEGER))); SCHEMA (name third, extends first, columns (third_id (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="first -> second -> third -> first", ), @@ -849,7 +849,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ SCHEMA (name base, columns (order_id (type INTEGER))); SCHEMA (name child, extends base, columns (ORDER_ID (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="redeclares inherited column 'ORDER_ID'", ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 2fa99d70f4..2afc430668 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -39,24 +39,24 @@ description="expands ordered typed model and macro test cases before macros", repo_files=base_repo_files() | { - "models/orders.sql": "MODEL ();\n\nSELECT 'open' AS status\n", - "models/customers.sql": "MODEL ();\n\nSELECT 1 AS customer_id\n", - "sources/raw.yml": "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 'open' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 1 AS customer_id\n", + "sources/raw.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n", "seeds/schema.yml": ( - "seeds:\n - name: country_codes\n columns:\n" + "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n" " - name: code\n type: VARCHAR\n" ), "seeds/country_codes.csv": "code\nGB\n", "models/_constants/offset.sql": "CONSTANT (name offset, value 2);\n", "models/_enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);\n", "functions/sql/add_one.sql": """ -FUNCTION (arguments (value INTEGER), returns INTEGER); +FUNCTION (description "Test function add_one.", arguments (value INTEGER), returns INTEGER); value + 1 """.strip() + "\n", "functions/sql/by_customer.sql": """ -FUNCTION ( +FUNCTION (description "Test function by_customer.", arguments (customer_id INTEGER), returns table (customer_id INTEGER) ); diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py index 0c4f939cf2..8e3379e8c7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py @@ -79,7 +79,7 @@ def test_given_windows_style_model_path_when_finding_path_default_then_returns_n model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / test_case.model_relative_path, relative_path=Path(test_case.model_relative_path), - contents="MODEL ();\n\nselect 1\n", + contents="MODEL (description 'Test model.');\n\nselect 1\n", header_values={}, header_column_locations={}, output_column_locations={}, @@ -118,7 +118,7 @@ def test_given_ambiguous_globs_when_finding_path_default_then_raises_structured_ model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / test_case.model_relative_path, relative_path=Path(test_case.model_relative_path), - contents="MODEL ();\n\nselect 1\n", + contents="MODEL (description 'Test model.');\n\nselect 1\n", header_values={}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index 13e530baf3..85a6f1f44c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -33,14 +33,16 @@ _DISABLED_PROJECT_FILE: str = _PROJECT_FILE + "\n[references]\nenforce_explicit = false\n" _HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py" _TASK_PATH: str = "python/tasks/export.py" -_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);' +_HOOKED_HEADER: str = ( + 'MODEL (description "Test model export.", post_hooks [python("refresh_lookup")]);' +) _MODELS: dict[str, str] = { "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"), "models/marts/orders.sql": gate_model(sql="SELECT 1 AS order_id", header=_HOOKED_HEADER), } _COUNTRY_SEED: dict[str, str] = { "seeds/country_codes.yml": ( - "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n - name: code\n type: VARCHAR\n" ), "seeds/country_codes.csv": "code\nGB\n", } @@ -51,9 +53,9 @@ _RAW_SOURCES: dict[str, str] = { "sources/raw.yml": ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" ), } diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py index 587c5b3e45..12f6f8acf7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py @@ -30,17 +30,17 @@ sql_validation = false """ _ADVISORY_PROJECT_FILE: str = _PROJECT_FILE + "\n[scopes]\nenforce_placement = false\n" -_ORDERS: str = "MODEL ();\nSELECT 1 AS order_id" -_CUSTOMERS: str = "MODEL ();\nSELECT 1 AS order_id" -_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n" -_SEED_YAML: str = ( - "seeds:\n - name: order_codes\n columns:\n - name: order_id\n type: INTEGER\n" +_ORDERS: str = "MODEL (description 'Test model.');\nSELECT 1 AS order_id" +_CUSTOMERS: str = "MODEL (description 'Test model.');\nSELECT 1 AS order_id" +_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n" ) -_TABLE_FUNCTION: str = "FUNCTION (returns table (order_id INTEGER));\nSELECT 1 AS order_id" +_SEED_YAML: str = "seeds:\n - name: order_codes\n description: Test seed order_codes.\n columns:\n - name: order_id\n type: INTEGER\n" +_TABLE_FUNCTION: str = "FUNCTION (description 'Test function.', returns table (order_id INTEGER));\nSELECT 1 AS order_id" _GENERIC_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL' _SCHEMA: str = "SCHEMA (name order_shape, columns (order_id (type INTEGER)));" -_SQL_HOOK: str = "HOOK ();\nSELECT 1" -_PYTHON_HOOK: str = "from sqlbuild.hooks import hook\n\n\n@hook\ndef mark(ctx):\n return None\n" +_SQL_HOOK: str = "HOOK (description 'Test hook.');\nSELECT 1" +_PYTHON_HOOK: str = "from sqlbuild.hooks import hook\n\n\n@hook\ndef mark(ctx):\n '''Test hook mark.'''\n return None\n" _SINGULAR_BASE_FILES: dict[str, str] = { "models/orders.sql": _ORDERS, "models/customers.sql": _CUSTOMERS, @@ -252,7 +252,7 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc "CONSTANT (name order_limit, value 10);" ), "models/staging/orders_stage.sql": ( - 'MODEL ();\nSELECT @const("order_limit") AS order_id' + 'MODEL (description "Test model orders_stage.");\nSELECT @const("order_limit") AS order_id' ), "models/marts/_sqlbuild/_audits/generic/order_check.sql": ( 'AUDIT ();\nSELECT * FROM __ref("@model") ' @@ -508,7 +508,7 @@ def test_given_invalid_named_declaration_layout_when_compiling_then_error_names_ "ENUM (name order_status, members [OPEN, CLOSED]);" ), "models/staging/orders_stage.sql": ( - 'MODEL ();\nSELECT @enum("order_status").OPEN AS status' + 'MODEL (description "Test model orders_stage.");\nSELECT @enum("order_status").OPEN AS status' ), "models/marts/_sqlbuild/_schemas/order_shape.sql": ( "SCHEMA (name order_shape, columns (status (type order_status)));" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index c1e5616c6a..d4134b031f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -37,8 +37,8 @@ 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_analysis = false\n' "sql_validation = false\n" ), - "models/orders_eu.sql": "MODEL ();\nSELECT 1 AS order_id", - "models/orders_us.sql": "MODEL ();\nSELECT 2 AS order_id", + "models/orders_eu.sql": "MODEL (description 'Test model orders_eu.');\nSELECT 1 AS order_id", + "models/orders_us.sql": "MODEL (description 'Test model orders_us.');\nSELECT 2 AS order_id", } @@ -392,7 +392,7 @@ def test_given_typed_macro_arguments_when_compiling_then_every_argument_is_a_dep _DEPENDENCY_PROJECT_FILES | { "macros/common.py": test_case.macro_file_contents, - "models/order_summary.sql": f"MODEL ();\n{test_case.sql}", + "models/order_summary.sql": f"MODEL (description 'Test model order_summary.');\n{test_case.sql}", }, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 20bbab00bf..f203b5f056 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -31,7 +31,7 @@ "[path_defaults.events]\ncursor_start = 20\n" ), "models/events/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy delete_insert\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy delete_insert\n" "cursor id\ncursor_type integer\ncursor_start 30\n);\n\nSELECT 1 AS id" ), }, @@ -74,7 +74,7 @@ def test_given_cursor_start_layers_when_building_compile_inputs_then_model_uses_ 'cursor_future_max_distance = "disabled"\n' ), "models/events/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy delete_insert\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy delete_insert\n" "cursor event_at\ncursor_type timestamp\ncursor_grain day\n" "cursor_start_max_ahead '0d'\ncursor_start_max_action cap\n" "cursor_future_max_distance '0d'\ncursor_future_action cap\n" @@ -117,7 +117,7 @@ def test_given_cursor_safety_layers_when_compiling_then_model_fields_use_model_p repo_files={ "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy merge\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy merge\n" "cursor_type integer\ncursor_start 100\n);\n\nSELECT 1 AS id" ), }, @@ -128,7 +128,7 @@ def test_given_cursor_safety_layers_when_compiling_then_model_fields_use_model_p repo_files={ "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy merge\ncursor id\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy merge\ncursor id\n" "cursor_type integer\nunique_key [id]\n" "cursor_start '3.14'\n);\n\nSELECT 1 AS id" ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index f320d89c0a..00bdbdcd28 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -128,7 +128,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (type VARCHAR), ), @@ -138,7 +138,7 @@ """.strip() + "\n", "models/staging/nested/orders_enriched.sql": """ -MODEL ( +MODEL (description "Test model orders_enriched.", materialized incremental, incremental_strategy delete_insert, cursor event_time, @@ -156,6 +156,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: country_code type: VARCHAR @@ -167,6 +168,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: public table: orders """.strip() @@ -175,7 +177,7 @@ selected_target=None, cli_vars={"shared": "cli", "cli_only": "present"}, run_id=None, - expected_model_schema_names=(None, "orders"), + expected_model_schema_names=("orders_enriched", "orders"), expected_model_config_values=( { "materialized": "incremental", @@ -220,11 +222,14 @@ ), BuildCompileInputsTestCase( description="allows models with no matching schema metadata", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target=None, cli_vars={}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -241,7 +246,7 @@ repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, contract enforced, columns ( @@ -295,7 +300,7 @@ repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -388,12 +393,12 @@ shared = "local" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target="prod", cli_vars={}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -444,12 +449,12 @@ target = "dev" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -481,12 +486,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -523,12 +528,12 @@ active = "dev" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -565,12 +570,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -604,12 +609,12 @@ warehouse = "dev_wh" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -645,12 +650,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"shared": "cli", "cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -672,14 +677,14 @@ repo_files=base_repo_files() | { "models/orders.sql": ( - "MODEL (materialized incremental, incremental_strategy append, " + "MODEL (description 'Test model orders.', materialized incremental, incremental_strategy append, " "full_refresh false);\n\nSELECT 1 AS id\n" ), }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -704,16 +709,16 @@ 'name = "demo"\nadapter = "duckdb"\n\n[defaults]\nfull_refresh = false\n' ), "models/a_incremental.sql": ( - "MODEL (materialized incremental, incremental_strategy append);" + "MODEL (description 'Test model a_incremental.', materialized incremental, incremental_strategy append);" "\n\nSELECT 1 AS id\n" ), - "models/b_view.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - "models/c_table.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/b_view.sql": "MODEL (description 'Test model b_view.', materialized view);\n\nSELECT 1 AS id\n", + "models/c_table.sql": "MODEL (description 'Test model c_table.', materialized table);\n\nSELECT 1 AS id\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, None, None), + expected_model_schema_names=("a_incremental", "b_view", "c_table"), expected_model_config_values=( { "full_refresh": False, @@ -746,12 +751,12 @@ '[defaults]\nmaterialized = "${model_kind}"\nfull_refresh = false\n' 'incremental_strategy = "append"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -800,7 +805,7 @@ """.strip() + "\n", "models/staging/orders.sql": ( - "MODEL (\n unique_key [order_id]," + "MODEL (description 'Test model orders.',\n unique_key [order_id]," "\n cursor event_time," "\n cursor_type timestamp," "\n cursor_grain second," @@ -810,7 +815,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -859,7 +864,7 @@ """.strip() + "\n", "models/staging/orders.sql": ( - "MODEL (\n cursor event_time," + "MODEL (description 'Test model orders.',\n cursor event_time," "\n cursor_type timestamp," "\n cursor_grain second," "\n);\n\nselect 1\n" @@ -868,7 +873,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -917,7 +922,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", row_diff_exclude_columns [run_id, loaded_at], row_diff_tolerances ( by_type ( @@ -941,7 +946,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "row_diff_exclude_columns": ("loaded_at", "run_id"), @@ -1001,7 +1006,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, alias '${CTX:model.name}_${CTX:run.target}', @@ -1023,7 +1028,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "dev_analytics_kevin", @@ -1078,7 +1083,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias 'orders_${CTX:run.target}', config ( run_label '${CTX:run.id}', @@ -1093,7 +1098,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "db_ci", @@ -1127,13 +1132,13 @@ schema = "${CTX:model.name}_schema" """.strip() + "\n", - "models/orders.sql": "MODEL ();\n\nselect 1\n", - "models/customers.sql": "MODEL ();\n\nselect 2\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nselect 2\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, None), + expected_model_schema_names=("customers", "orders"), expected_model_config_values=( {"schema": "customers_schema"}, {"schema": "orders_schema"}, @@ -1165,7 +1170,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", cursor event_time, cursor_type timestamp, cursor_grain second, @@ -1179,7 +1184,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -1316,12 +1321,12 @@ schema = "preserve" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "analytics_team_prod", @@ -1381,7 +1386,7 @@ def grant_select(target: str) -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias orders_dev, post_hooks [inline_sql('@grant_select("@@CTX:destination.qualified")')], ); @@ -1412,6 +1417,7 @@ def grant_select(target: str) -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. table: orders """.strip() + "\n", @@ -1423,7 +1429,7 @@ def grant_select(target: str) -> str: "optional_suffix": None, }, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "schema": "marts", @@ -1504,7 +1510,7 @@ def grant_select(target: str) -> str: } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -1525,7 +1531,7 @@ def grant_select(target: str) -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_path_defaults=(None,), expected_seed_names=(), @@ -1583,7 +1589,7 @@ def source_columns() -> str: """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [inline_sql("insert into @@audit_schema.load_log select '@@ENV:USER_NAME'")], columns (order_id (audits [source_filter])), ); @@ -1595,7 +1601,7 @@ def source_columns() -> str: """.strip() + "\n", "functions/sql/is_large_order.sql": """ -FUNCTION ( +FUNCTION (description "Test function is_large_order.", arguments (amount INTEGER), returns BOOLEAN, ); @@ -1635,10 +1641,12 @@ def source_columns() -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT @source_columns(), '@@ENV:USER_NAME' AS loaded_by FROM @@raw_schema.raw_orders - name: raw_table + description: Test source raw_table. database: ${raw_database} schema: ${raw_schema} table: orders_${ENV:USER_NAME} @@ -1747,7 +1755,7 @@ def source_columns() -> str: """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias orders_dev, post_hooks [ inline_sql("select '@@CTX:destination.qualified' as target"), @@ -1765,6 +1773,7 @@ def source_columns() -> str: @hook def notify(ctx, message): + '''Test hook notify.''' return None """.strip() + "\n", @@ -1772,7 +1781,7 @@ def notify(ctx, message): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "schema": "marts", @@ -1816,7 +1825,7 @@ def notify(ctx, message): """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", post_hooks [sql("record_access", role: "O'Brien")], ); @@ -1834,7 +1843,7 @@ def notify(ctx, message): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "analytics", @@ -1873,7 +1882,7 @@ def notify(ctx, message): repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [ python("defaulted", channel: "alerts"), python("flexible", extra: "value"), @@ -1891,21 +1900,25 @@ def notify(ctx, message): @hook def defaulted(context, channel, message="ready"): + '''Test hook defaulted.''' return None @hook def flexible(ctx, **kwargs): + '''Test hook flexible.''' return None @hook def context_named(hook_context, message): + '''Test hook context_named.''' return None @hook def optional_positional_only(ctx, channel="alerts", /): + '''Test hook optional_positional_only.''' return None """.strip() + "\n", @@ -1913,7 +1926,7 @@ def optional_positional_only(ctx, channel="alerts", /): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "pre_hooks": [ @@ -1938,7 +1951,7 @@ def optional_positional_only(ctx, channel="alerts", /): repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", post_hooks [ python("notify", message: "@@CTX:destination.qualified", macro_text: "@format_message()"), ], @@ -1953,6 +1966,7 @@ def optional_positional_only(ctx, channel="alerts", /): @hook def notify(ctx, message, macro_text): + '''Test hook notify.''' return None """.strip() + "\n", @@ -1960,7 +1974,7 @@ def notify(ctx, message, macro_text): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "post_hooks": [ @@ -2013,7 +2027,7 @@ def status_match(column_name: str, status: str) -> str: """.strip() + "\n", "functions/sql/is_completed_order.sql": """ -FUNCTION ( +FUNCTION (description "Test function is_completed_order.", database ${udf_database}, schema ${udf_schema}, arguments (order_status ${status_type}), @@ -2074,10 +2088,10 @@ def status_match(column_name: str, status: str) -> str: """.strip() + "\n", "models/fact_orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(7)\n' + 'MODEL (description "Test model fact_orders.");\n\nSELECT * FROM __table_fn("customer_orders")(7)\n' ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -2094,7 +2108,7 @@ def status_match(column_name: str, status: str) -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("fact_orders",), expected_model_config_values=({},), expected_model_query_sqls=('SELECT * FROM __table_fn("customer_orders")(7)',), expected_model_path_defaults=(None,), @@ -2131,7 +2145,7 @@ def status_match(column_name: str, status: str) -> str: | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", returns table ( order_id INTEGER, status VARCHAR @@ -2188,7 +2202,7 @@ def project_columns() -> str: return "order_id" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (name "first"); @@ -2226,8 +2240,10 @@ def project_columns() -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. table: orders - name: raw_customers + description: Test source raw_customers. table: customers """.strip() + "\n", @@ -2235,7 +2251,7 @@ def project_columns() -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2316,6 +2332,7 @@ def project_columns() -> str: packages=["faker"], ) def main(order_status): + '''Test function main.''' return order_status == "completed" """.strip() + "\n", @@ -2336,6 +2353,7 @@ def main(order_status): expected_sql_function_body_sqls=( """ def main(order_status): + \"\"\"Test function main.\"\"\" return order_status == 'completed' """.strip(), ), @@ -2362,7 +2380,7 @@ def main(order_status): repo_files=base_repo_files() | { "models/marts/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [model_not_null], columns ( order_id (audits [column_not_null]), @@ -2375,6 +2393,7 @@ def main(order_status): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - source_not_null columns: @@ -2440,7 +2459,7 @@ def main(order_status): repo_files=base_repo_files() | { "models/marts/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (nullable false, audits [not_null, unique]), status (audits [accepted_values (values ["placed", "completed"])]), @@ -2454,10 +2473,11 @@ def main(order_status): select 1 as order_id, 'placed' as status, null as customer_id """.strip() + "\n", - "models/marts/customers.sql": "MODEL ();\n\nselect 1 as customer_id\n", + "models/marts/customers.sql": "MODEL (description 'Test model customers.');\n\nselect 1 as customer_id\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. columns: - name: order_id audits: @@ -2468,7 +2488,7 @@ def main(order_status): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, "orders"), + expected_model_schema_names=("customers", "orders"), expected_model_config_values=({}, {}), expected_model_query_sqls=( "select 1 as customer_id", @@ -2509,7 +2529,7 @@ def main(order_status): description="skips generic audit definitions as direct executable audits", repo_files=base_repo_files() | { - "models/marts/orders.sql": "MODEL ();\n\nselect 1\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "audits/generic/custom_check.sql": """ AUDIT (); @@ -2520,7 +2540,7 @@ def main(order_status): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2538,13 +2558,13 @@ def main(order_status): description="warns when project generic audit shadows built-in audit", repo_files=base_repo_files() | { - "models/marts/orders.sql": "MODEL ();\n\nselect 1\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "audits/generic/not_null.sql": "AUDIT ();\n\nSELECT 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2564,11 +2584,14 @@ def main(order_status): ), BuildCompileInputsTestCase( description="generates clickstate style run ids when none are provided", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2592,12 +2615,12 @@ def main(order_status): sql_validation = false """.strip() + "\n", - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2623,12 +2646,12 @@ def main(order_status): sql_analysis = false """.strip() + "\n", - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2646,13 +2669,13 @@ def main(order_status): description="allows invalid sql when no_sql_validation flag is set", repo_files=base_repo_files() | { - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", no_sql_validation=True, - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2690,14 +2713,14 @@ def main(order_status): """.strip() + "\n", "models/fact_orders.sql": ( - 'MODEL ();\n\nselect * from __dbt_ref("stg_orders") ' + 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("stg_orders") ' 'union all select * from __dbt_ref("stripe", "orders")\n' ), }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("fact_orders",), expected_model_config_values=({},), expected_model_query_sqls=( 'select * from __dbt_ref("stg_orders") ' @@ -3056,9 +3079,9 @@ def test_given_preserved_target_when_building_functions_then_logical_namespaces_ """.strip() + "\n", "functions/sql/defaulted.sql": ( - "FUNCTION (returns INTEGER, tags [reference, daily]);\n\n1\n" + "FUNCTION (description 'Test function defaulted.', returns INTEGER, tags [reference, daily]);\n\n1\n" ), - "models/model_defaulted.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/model_defaulted.sql": "MODEL (description 'Test model model_defaulted.');\n\nSELECT 1 AS id\n", "functions/python/explicit.py": """ from sqlbuild.functions import udf @@ -3070,6 +3093,7 @@ def test_given_preserved_target_when_building_functions_then_logical_namespaces_ schema="function_schema", ) def main(): + '''Test function main.''' return 1 """.strip() + "\n", @@ -3078,6 +3102,7 @@ def main(): @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 1 """.strip() + "\n", @@ -3219,7 +3244,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov """.strip() + "\n", "models/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __table_fn("missing_orders")(1) """.strip() @@ -3235,7 +3260,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER, customer_id INTEGER) ); @@ -3253,9 +3278,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": ('MODEL ();\n\nSELECT * FROM __table_fn("order_total")(1)\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("order_total")(1)\n' + ), "functions/sql/order_total.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_total.", arguments (order_id INTEGER), returns INTEGER ); @@ -3275,9 +3302,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sqlbuild_project.toml": ( 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")\n' + ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3298,7 +3327,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT * FROM __table_fn('customer_orders')(1)\n" + "MODEL (description 'Test model orders.');\n\nSELECT * FROM __table_fn('customer_orders')(1)\n" ), }, selected_target=None, @@ -3313,10 +3342,10 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(1, 2)\n' + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")(1, 2)\n' ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3337,7 +3366,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(1,)\n' + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")(1,)\n' ), }, selected_target=None, @@ -3350,7 +3379,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3379,7 +3408,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __source("missing_source") """.strip() @@ -3394,7 +3423,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, contract enforced, columns ( @@ -3419,7 +3448,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -3444,7 +3473,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -3485,7 +3514,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __dbt_ref("stg_orders") """.strip() @@ -3503,7 +3532,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "functions/sql/orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function orders.", returns table (order_id INTEGER) ); @@ -3543,6 +3572,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - source_not_null: source: other_source @@ -3564,7 +3594,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (nullable true, audits [not_null]), ), @@ -3587,6 +3617,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - missing_definition """.strip() @@ -3615,7 +3646,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body lacks top level test ctes", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3631,10 +3662,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body lacks ceremonial select one", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3659,7 +3691,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown source mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3682,7 +3714,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown seed mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3722,10 +3754,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw__orders + description: Test source raw__orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3766,11 +3799,12 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "seeds/countries.csv": "country_code,country_name\nUS,United States\n", "seeds/schema.yml": """ seeds: - name: countries + description: Test seed countries. columns: - name: country_code type: VARCHAR @@ -3800,10 +3834,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown macro mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3834,6 +3869,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3858,10 +3894,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body uses a reserved helper cte name", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3914,7 +3951,7 @@ def project_columns() -> str: return "customer_id" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -3925,7 +3962,7 @@ def project_columns() -> str: repo_files=base_repo_files() | { "macros/orders.py": "def DailyOrders():\n return 'order_id'\n", - "models/staging/orders.sql": "MODEL ();\n\nselect @DailyOrders()\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect @DailyOrders()\n", }, selected_target=None, run_id=None, @@ -3944,7 +3981,7 @@ def dynamic_schema() -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", schema '@dynamic_schema()', ); @@ -3958,7 +3995,10 @@ def dynamic_schema() -> str: ), BuildCompileInputsErrorTestCase( description="raises when the selected target does not exist", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target="missing", run_id=None, expected_error_fragment="Unknown target 'missing'", @@ -3972,7 +4012,7 @@ def dynamic_schema() -> str: target = "missing" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -3989,7 +4029,7 @@ def dynamic_schema() -> str: default_target = "missing" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4008,7 +4048,7 @@ def dynamic_schema() -> str: user = "${missing}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4026,7 +4066,7 @@ def dynamic_schema() -> str: path = "${ENV:SQLBUILD_DB_PATH}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4044,7 +4084,7 @@ def dynamic_schema() -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", schema '${CTX:this}', ); @@ -4074,7 +4114,7 @@ def dynamic_schema() -> str: schema = "dev_${CTX:destination.missing}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4097,7 +4137,7 @@ def dynamic_schema() -> str: database = "dev_${CTX:model.database}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4118,7 +4158,7 @@ def dynamic_schema() -> str: second = "${first}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4141,7 +4181,7 @@ def dynamic_schema() -> str: schema = "${SQLBUILD:user}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4159,7 +4199,7 @@ def dynamic_schema() -> str: path = "${CTX:schema}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4169,7 +4209,7 @@ def dynamic_schema() -> str: description="raises when model query sql has invalid syntax", repo_files=base_repo_files() | { - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, run_id=None, @@ -4180,7 +4220,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks ['SELECT 1']);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model broken.', post_hooks ['SELECT 1']);\n\nSELECT 1 AS id\n" ), }, selected_target=None, @@ -4192,7 +4232,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks [inline_sql('')]);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model broken.', post_hooks [inline_sql('')]);\n\nSELECT 1 AS id\n" ), }, selected_target=None, @@ -4209,9 +4249,9 @@ def dynamic_schema() -> str: 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/staging/broken.sql": ( - 'MODEL (post_hooks [sql("dynamic", statement: "")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [sql("dynamic", statement: "")]);\n\nSELECT 1 AS id\n' ), - "hooks/sql/dynamic.sql": "HOOK ();\n\n@statement\n", + "hooks/sql/dynamic.sql": "HOOK (description 'Test hook dynamic.');\n\n@statement\n", }, selected_target=None, run_id=None, @@ -4224,7 +4264,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks [" + "MODEL (description 'Test model broken.', post_hooks [" "inline_sql('GRANT SELECT ON ${CTX:destination.qualified} TO analyst')" "]);\n\n" "SELECT 1 AS id\n" @@ -4242,7 +4282,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), }, selected_target=None, @@ -4256,7 +4296,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("known"), python("missing")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("known"), python("missing")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4264,6 +4304,7 @@ def dynamic_schema() -> str: @hook def known(ctx): + '''Test hook known.''' return None """.strip() + "\n", @@ -4279,7 +4320,7 @@ def known(ctx): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (pre_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", pre_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4287,6 +4328,7 @@ def known(ctx): @hook def notify(ctx, channel): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4302,7 +4344,7 @@ def notify(ctx, channel): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify", unknown: "value")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify", unknown: "value")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4310,6 +4352,7 @@ def notify(ctx, channel): @hook def notify(ctx): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4325,7 +4368,7 @@ def notify(ctx): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4333,6 +4376,7 @@ def notify(ctx): @hook def notify(ctx, channel, /): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4348,7 +4392,7 @@ def notify(ctx, channel, /): description="raises when model header tags is a string instead of list", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL (tags nightly);\n\nSELECT 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.', tags nightly);\n\nSELECT 1\n", }, selected_target=None, run_id=None, @@ -4358,7 +4402,7 @@ def notify(ctx, channel, /): description="raises when dbt ref has no discovered manifest", repo_files=base_repo_files() | { - "models/fact_orders.sql": 'MODEL ();\n\nselect * from __dbt_ref("orders")\n', + "models/fact_orders.sql": 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("orders")\n', }, selected_target=None, run_id=None, @@ -4389,7 +4433,7 @@ def notify(ctx, channel, /): } """.strip() + "\n", - "models/fact_orders.sql": 'MODEL ();\n\nselect * from __dbt_ref("orders")\n', + "models/fact_orders.sql": 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("orders")\n', }, selected_target=None, run_id=None, @@ -4413,7 +4457,7 @@ def notify(ctx, channel, /): } """.strip() + "\n", - "models/orders.sql": "MODEL ();\n\nselect 1\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4457,7 +4501,7 @@ def test_given_attachment_conflicts_when_building_compile_inputs_then_it_raises_ repo_files=base_repo_files() | { "models/staging/orders.sql": ( - 'MODEL (post_hooks [python("notify", channel: "alerts")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model orders.", post_hooks [python("notify", channel: "alerts")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from pathlib import Path @@ -4466,6 +4510,7 @@ def test_given_attachment_conflicts_when_building_compile_inputs_then_it_raises_ @hook def notify(ctx, channel): + '''Test hook notify.''' Path(__file__).with_name("executed.marker").write_text("executed", encoding="utf-8") """.strip() + "\n", @@ -4648,7 +4693,7 @@ def test_given_retention_layers_when_building_model_config_then_precedence_is_re ) ), "models/orders.sql": ( - "MODEL (materialized table, time_travel_retention 30d);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model orders.', materialized table, time_travel_retention 30d);\n\nSELECT 1 AS id\n" ), }, expected_desired_days=30, @@ -4685,7 +4730,7 @@ def test_given_authored_retention_header_when_building_inputs_then_typed_policy_ repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -4741,7 +4786,7 @@ def test_given_snapshot_contract_schema_change_conflict_when_building_then_error "seeds/waffle_types.csv": "waffle_type_id,waffle_name\n1,Classic\n", "seeds/schema.yml": ( "seeds:\n" - " - name: waffle_types\n" + " - name: waffle_types\n description: Test seed waffle_types.\n" " columns:\n" " - name: waffle_type_id\n" " type: INTEGER\n" @@ -4749,7 +4794,7 @@ def test_given_snapshot_contract_schema_change_conflict_when_building_then_error " type: VARCHAR\n" ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT waffle_type_id FROM __seed("waffle_types")' + 'MODEL (description "Test model orders.");\n\nSELECT waffle_type_id FROM __seed("waffle_types")' ), }, expected_model_count=1, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index fd95957051..d162c1b057 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -28,7 +28,7 @@ description="resolved connection override becomes the effective connection", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, resolved_connection={"database": "/abs/warehouse.duckdb", "schema": "analytics"}, expected_effective_connection={ @@ -40,7 +40,7 @@ description="without override the merged project connection is used", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, resolved_connection=None, expected_effective_connection={"database": "warehouse.duckdb"}, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 914534f838..7738441e93 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -61,7 +61,7 @@ def test_given_nested_type_key_when_locating_declared_type_then_returns_top_leve def test_given_dynamic_family_when_generating_contract_then_does_not_freeze_runtime_members( test_case: DynamicContractAdoptionTestCase, ) -> None: - authored_sql: str = """MODEL ( + authored_sql: str = """MODEL (description "Test model.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py index 18b847af75..f6bc24dc9d 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py @@ -353,7 +353,7 @@ def test_given_deferred_output_locations_when_contract_fails_then_locates_author type_enforcement=True, contract="enforced", authored_sql=( - "MODEL (columns (order_id (type INTEGER)));\n" + "MODEL (description 'Test model.', columns (order_id (type INTEGER)));\n" "SELECT\n" " 1 AS order_id,\n" " 2 AS amount\n" diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 73797d57f1..77a59535d7 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -958,7 +958,7 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc DiscoveredSqlHookFile( file_path=Path("/project/hooks/sql/record_access.sql"), relative_path=Path("hooks/sql/record_access.sql"), - contents="HOOK ();\n\nSELECT @'role'\n", + contents="HOOK (description 'Test hook record_access.');\n\nSELECT @'role'\n", header_values={"description": "Record access"}, sql_body="SELECT @'role'", name="record_access", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index ffb78f706f..880fecbf85 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -54,8 +54,8 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ tmp_path, { "sqlbuild_project.toml": 'name = "source_batch"\nadapter = "duckdb"\n', - "sources/orders.yml": "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n", - "models/combined.sql": 'MODEL ();\nSELECT o.id FROM __source("orders") o JOIN __source("customers") c ON o.id = c.id', + "sources/orders.yml": "sources:\n - name: orders\n description: Test source orders.\n table: orders\n - name: customers\n description: Test source customers.\n table: customers\n", + "models/combined.sql": 'MODEL (description "Test model combined.");\nSELECT o.id FROM __source("orders") o JOIN __source("customers") c ON o.id = c.id', }, ) adapter: DuckDbAdapter = DuckDbAdapter() @@ -109,10 +109,10 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found { "sqlbuild_project.toml": 'name = "source_case"\nadapter = "duckdb"\n', "sources/orders.yml": ( - "sources:\n - name: orders\n" + "sources:\n - name: orders\n description: Test source orders.\n" f" schema: {test_case.declared_schema}\n table: orders\n" ), - "models/open_orders.sql": 'MODEL ();\nSELECT id FROM __source("orders")', + "models/open_orders.sql": 'MODEL (description "Test model open_orders.");\nSELECT id FROM __source("orders")', }, ) adapter: DuckDbAdapter = DuckDbAdapter() diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index 719197ed47..62a7809ca1 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -323,6 +323,7 @@ def audit_message(): @hook(name="before_orders") def before_orders(ctx): + '''Test hook before_orders.''' return audit_message() """.strip() + "\n", diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 5e62b7decd..e420ec69c1 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -41,7 +41,7 @@ from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig SCOPE_CACHE_PROJECT: str = 'name = "demo"\nadapter = "duckdb"\n' -SCOPE_CACHE_MODEL: str = "MODEL();\nSELECT 1 AS id\n" +SCOPE_CACHE_MODEL: str = "MODEL(description 'Test model.');\nSELECT 1 AS id\n" def unused_declaration_index(*, enforce_placement: bool) -> ScopeIndex: @@ -145,7 +145,7 @@ def discovered_model( return DiscoveredSqlModelFile( file_path=relative, relative_path=relative, - contents="MODEL(); SELECT 1", + contents="MODEL(description 'Test model.'); SELECT 1", header_values={"name": name}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index 33fb48fe73..d7167b9f2b 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -159,7 +159,11 @@ def test_given_no_cache_when_loading_twice_then_cache_is_bypassed( "test_case", ( FingerprintMutationCase( - "model content", "models/orders.sql", _MODEL, "MODEL();\nSELECT 2 AS id\n", True + "model content", + "models/orders.sql", + _MODEL, + "MODEL(description 'Test model orders.');\nSELECT 2 AS id\n", + True, ), FingerprintMutationCase( "declaration rendering", @@ -314,7 +318,7 @@ def test_given_target_aware_python_composition_when_switching_target_then_depend _PROJECT + '\ndefault_target = "dev"\n[targets.dev]\n[targets.prod]\n' ), "sqlbuild_local.toml": 'target = "dev"\n', - "models/orders.sql": "MODEL();\nSELECT @choose() AS value\n", + "models/orders.sql": "MODEL(description 'Test model orders.');\nSELECT @choose() AS value\n", "macros/choose.py": ( "from macros.dev import dev_macro\n" "from macros.prod import prod_macro\n\n" @@ -535,7 +539,7 @@ def test_given_placement_invalid_project_when_loading_then_complete_diagnostics_ { "sqlbuild_project.toml": _PROJECT, "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ) @@ -563,7 +567,7 @@ def test_given_cached_placement_error_when_disabling_enforcement_then_cache_retu { "sqlbuild_project.toml": _PROJECT + "\n[scopes]\nenforce_placement = true\n", "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ) enforced: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) @@ -626,7 +630,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="SQL functions retain valid siblings", files={ - "functions/sql/valid.sql": "FUNCTION (returns INTEGER);\n1", + "functions/sql/valid.sql": "FUNCTION (description 'Test function valid.', returns INTEGER);\n1", "functions/sql/broken.sql": "FUNCTION (\nSELECT 1", }, expected_kind=ResourceKind.FUNCTION, @@ -635,7 +639,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="SQL hooks retain valid siblings", files={ - "hooks/sql/valid.sql": "HOOK ();\nSELECT 1", + "hooks/sql/valid.sql": "HOOK (description 'Test hook valid.');\nSELECT 1", "hooks/sql/broken.sql": "HOOK (\nSELECT 1", }, expected_kind=DeclarationKind.SQL_HOOK, @@ -653,7 +657,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="sources retain valid siblings", files={ - "sources/valid.yml": "sources:\n - name: raw_orders\n table: orders\n", + "sources/valid.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n", "sources/broken.yml": "sources: [\n", }, expected_kind=ResourceKind.SOURCE, diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index 55b80c10a5..d0c43013f9 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -123,7 +123,7 @@ def build_clone_model_entry(*, schema: str, name: str) -> ModelPlanEntry: ), fingerprint_query_sql="SELECT 1", resolved_sql="SELECT 1", - logical_ddl="MODEL (materialized table);", + logical_ddl="MODEL (description 'Test model.', materialized table);", ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 9c576bb450..461c921660 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -265,7 +265,7 @@ def build_compiled_scenario(*, name: str) -> CompiledSqlScenario: scenario_file=DiscoveredSqlScenarioFile( file_path=Path(f"tests/scenarios/{name}.sql"), relative_path=Path(f"tests/scenarios/{name}.sql"), - contents="SCENARIO ();", + contents="SCENARIO (description 'Test scenario.');", header_values={}, sql_body="", name=name, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index efcfe9c8f6..dcd6964ca9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -423,7 +423,7 @@ def build_compiled_project_with_model_specs( model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / relative_path, relative_path=relative_path, - contents=f"MODEL ();\n\n{sql}\n", + contents=f"MODEL (description 'Test model.');\n\n{sql}\n", header_values={}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py index 00131498a8..3ec89456f7 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py @@ -19,28 +19,28 @@ HeaderAuditWrapTestCase( "audits then their arguments go one per line", 50, - 'MODEL (\n audits [expression_is_true (name "positive", expression "total > 0")],\n);\n', - "MODEL (\n audits [\n expression_is_true (\n" + 'MODEL (\n description "Test model.",\n audits [expression_is_true (name "positive", expression "total > 0")],\n);\n', + 'MODEL (\n description "Test model.",\n audits [\n expression_is_true (\n' ' name "positive",\n expression "total > 0",\n ),\n ],\n);\n', ), HeaderAuditWrapTestCase( "quoted values are never split", 30, - 'MODEL (\n audits [expression_is_true (expression "a, b, c, d > 0")],\n);\n', - "MODEL (\n audits [\n expression_is_true (\n" + 'MODEL (\n description "Test model.",\n audits [expression_is_true (expression "a, b, c, d > 0")],\n);\n', + 'MODEL (\n description "Test model.",\n audits [\n expression_is_true (\n' ' expression "a, b, c, d > 0",\n ),\n ],\n);\n', ), HeaderAuditWrapTestCase( "a line with a comment is left alone", 30, - "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", - "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", + 'MODEL (\n description "Test model.",\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n', + 'MODEL (\n description "Test model.",\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n', ), HeaderAuditWrapTestCase( "lists outside audits are left alone", 30, - "MODEL (\n tags [orders, customers, products, inventory],\n);\n", - "MODEL (\n tags [orders, customers, products, inventory],\n);\n", + 'MODEL (\n description "Test model.",\n tags [orders, customers, products, inventory],\n);\n', + 'MODEL (\n description "Test model.",\n tags [orders, customers, products, inventory],\n);\n', ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py index 8d39b52324..44546af55e 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py @@ -17,7 +17,7 @@ [ ScanHeadersTestCase( description="finds model header", - contents="MODEL (\n materialized table\n);\nSELECT 1\n", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\n', expected_kinds=("MODEL",), ), ScanHeadersTestCase( @@ -57,7 +57,7 @@ ScanHeadersTestCase( description="ignores header keyword used as a nested column name", contents=( - "MODEL (\n columns (\n object_id (type INTEGER),\n" + 'MODEL (\n description "Test model.",\n columns (\n object_id (type INTEGER),\n' " SCHEMA (type VARCHAR(100)),\n ),\n);\nSELECT 1 AS object_id\n" ), expected_kinds=("MODEL",), @@ -65,7 +65,7 @@ ScanHeadersTestCase( description="ignores header-like text inside comments", contents=( - '-- TEST ();\n/*\nMODEL (not a header);\n*/\nMODEL (description "ok");\nSELECT 1\n' + '-- TEST ();\n/*\nMODEL (description "Test model.", not a header);\n*/\nMODEL (description "ok");\nSELECT 1\n' ), expected_kinds=("MODEL",), ), @@ -86,7 +86,7 @@ ), ScanHeadersTestCase( description="standard SQL backslash does not escape closing quote", - contents="SCENARIO ();\nSELECT 'C:\\temp\\'\nTEST ();\nSELECT 1\n", + contents="SCENARIO (description \"Test scenario.\");\nSELECT 'C:\\temp\\'\nTEST ();\nSELECT 1\n", expected_kinds=("SCENARIO", "TEST"), ), ], @@ -104,7 +104,7 @@ def test_given_contents_when_scanning_headers_then_kinds_match_expected( [ ScanHeadersTestCase( description="first header only", - contents="MODEL ();\nSELECT 1\nTEST ();\nSELECT 2\n", + contents='MODEL (description "Test model.");\nSELECT 1\nTEST ();\nSELECT 2\n', expected_kinds=("MODEL",), ) ], @@ -125,13 +125,13 @@ def test_given_single_header_resource_when_scanning_first_only_then_body_is_not_ [ HeaderSpanTextTestCase( description="span covers header keyword through terminator", - contents="MODEL (\n materialized table\n);\nSELECT 1\n", - expected_span_text="MODEL (\n materialized table\n);", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\n', + expected_span_text='MODEL (\n description "Test model.",\n materialized table\n);', ), HeaderSpanTextTestCase( description="span without terminator ends at close paren", - contents="MODEL (\n materialized table\n)\nSELECT 1\n", - expected_span_text="MODEL (\n materialized table\n)", + contents='MODEL (\n description "Test model.",\n materialized table\n)\nSELECT 1\n', + expected_span_text='MODEL (\n description "Test model.",\n materialized table\n)', ), HeaderSpanTextTestCase( description="escaped quotes and quoted parens stay inside the span", @@ -165,7 +165,7 @@ def test_given_model_file_when_scanning_then_span_covers_header_and_terminator( [ SqlBodyRangesTestCase( description="bodies exclude headers and keep sql text", - contents="MODEL (\n materialized table\n);\nSELECT 1\nTEST ();\nSELECT 2\n", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\nTEST ();\nSELECT 2\n', expected_fragments=("SELECT 1", "SELECT 2"), excluded_fragments=("MODEL", "TEST ()"), ), diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index b81b9a9119..e85fc81211 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -514,12 +514,12 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select model_dir: Path = tmp_path / "models" model_dir.mkdir() model_dir.joinpath("stg_orders.sql").write_text( - "MODEL (columns (order_id (type INTEGER), status (type VARCHAR)));\n" + 'MODEL (description "Test model stg_orders.", columns (order_id (type INTEGER), status (type VARCHAR)));\n' "SELECT 1 AS order_id, 'open' AS status\n", encoding="utf-8", ) model_dir.joinpath("orders.sql").write_text( - "MODEL (columns (order_id (type INTEGER), status (type VARCHAR)));\n" + 'MODEL (description "Test model orders.", columns (order_id (type INTEGER), status (type VARCHAR)));\n' 'SELECT order_id, status FROM __ref("stg_orders")\n', encoding="utf-8", ) diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index c1093a4554..a930e53846 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -191,7 +191,7 @@ def test_given_provider_class_imported_through_alias_when_calling_then_guides_us provider_file: Path = tmp_path / "providers" / "marker.py" provider_file.parent.mkdir(parents=True) provider_file.write_text( - "from sqlbuild.providers import Provider\n\nclass MarkerProvider(Provider):\n pass\n", + "from sqlbuild.providers import Provider\n\nclass MarkerProvider(Provider):\n '''Test provider MarkerProvider.'''\n pass\n", encoding="utf-8", ) old_path: list[str] = list(sys.path) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index 6530d6355b..bfce6b065f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -33,7 +33,7 @@ rule_case=RuleCase( description="wrong configured domain faults", source=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "WITH final AS (SELECT 1 AS id)\n" "SELECT id FROM final\n" ), @@ -65,7 +65,7 @@ def test_given_rule_case_when_evaluating_then_runs_real_pipeline( rule_case=RuleCase( description="wrong configured domain faults", source=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "WITH final AS (SELECT 1 AS id)\n" "SELECT id FROM final\n" ), @@ -129,7 +129,7 @@ def test_given_same_custom_rule_when_using_public_and_native_paths_then_faults_h description="module-level state leaking between evaluations fails the harness", rule_case=RuleCase( description="first evaluation reports nothing", - source="MODEL (materialized table);\n\nSELECT 1 AS id\n", + source="MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id\n", path="models/mart/commerce__mart__orders.sql", expected_finding_count=0, ), diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 8f1ac856b1..9024e609d5 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -57,7 +57,7 @@ def test_given_sql_analysis_state_when_running_rules_then_type_proof_rules_follo models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (order_id (type INTEGER), status (type VARCHAR)),\n" ");\n" From 23f162054a1f3374fff3b2b9ec75e2e9629bfdda Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:53:22 +0100 Subject: [PATCH 3/4] docs: document required descriptions for every named resource Describe the P010 requirement, where each resource kind declares its description, the new FUNCTION and @udf description fields, loader and provider docstrings, dlt resource descriptions and the format --check fault. Add descriptions to every documentation and README example and regenerate the agent skill references. --- README.md | 1 + src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 4 ++ .../sqlbuild/references/docs/cli/format.md | 8 ++++ .../sqlbuild/references/docs/cli/scope.md | 2 +- .../references/docs/concepts/audits.md | 6 +++ .../adding-sqlbuild-models.md | 3 +- .../sqlbuild/references/docs/concepts/diff.md | 3 ++ .../docs/concepts/enums/model-contracts.md | 1 + .../references/docs/concepts/functions.md | 8 +++- .../references/docs/concepts/incremental.md | 13 ++++++ .../references/docs/concepts/macros.md | 2 + .../docs/concepts/model-private-values.md | 2 + .../docs/concepts/models/column-migrations.md | 1 + .../docs/concepts/models/configuration.md | 40 ++++++++++++++++++- .../docs/concepts/models/contracts.md | 2 + .../references/docs/concepts/models/hooks.md | 1 + .../docs/concepts/models/hooks/python.md | 6 ++- .../docs/concepts/models/hooks/sql.md | 10 +++-- .../docs/concepts/models/materializations.md | 7 +++- .../docs/concepts/models/migrations.md | 1 + .../docs/concepts/models/type-enforcement.md | 1 + .../docs/concepts/python-nodes/assets.md | 6 +++ .../docs/concepts/python-nodes/checks.md | 4 ++ .../docs/concepts/python-nodes/factories.md | 4 ++ .../docs/concepts/python-nodes/loaders.md | 13 ++++++ .../docs/concepts/python-nodes/overview.md | 3 +- .../docs/concepts/python-nodes/providers.md | 12 ++++++ .../concepts/python-nodes/sql-references.md | 4 ++ .../docs/concepts/python-nodes/tasks.md | 7 ++++ .../rules/configuration-and-selection.md | 5 +++ .../references/docs/concepts/scenarios.md | 2 +- .../references/docs/concepts/seeds.md | 5 ++- .../references/docs/concepts/snapshots.md | 9 +++++ .../references/docs/concepts/sources.md | 8 +++- .../docs/integrations/dagster-reference.md | 1 + .../references/docs/integrations/dlt.md | 6 +++ .../references/docs/integrations/ingestr.md | 3 ++ website/src/content/docs/docs/cli/format.mdx | 8 ++++ website/src/content/docs/docs/cli/scope.mdx | 2 +- .../src/content/docs/docs/concepts/audits.mdx | 6 +++ .../adding-sqlbuild-models.mdx | 3 +- .../src/content/docs/docs/concepts/diff.mdx | 3 ++ .../docs/concepts/enums/model-contracts.mdx | 1 + .../content/docs/docs/concepts/functions.mdx | 8 +++- .../docs/docs/concepts/incremental.mdx | 13 ++++++ .../src/content/docs/docs/concepts/macros.mdx | 2 + .../docs/concepts/model-private-values.mdx | 2 + .../concepts/models/column-migrations.mdx | 1 + .../docs/concepts/models/configuration.mdx | 39 +++++++++++++++++- .../docs/docs/concepts/models/contracts.mdx | 2 + .../docs/docs/concepts/models/hooks.mdx | 1 + .../docs/concepts/models/hooks/python.mdx | 6 ++- .../docs/docs/concepts/models/hooks/sql.mdx | 10 +++-- .../docs/concepts/models/materializations.mdx | 7 +++- .../docs/docs/concepts/models/migrations.mdx | 1 + .../docs/concepts/models/type-enforcement.mdx | 1 + .../docs/concepts/python-nodes/assets.mdx | 6 +++ .../docs/concepts/python-nodes/checks.mdx | 4 ++ .../docs/concepts/python-nodes/factories.mdx | 4 ++ .../docs/concepts/python-nodes/loaders.mdx | 13 ++++++ .../docs/concepts/python-nodes/overview.mdx | 3 +- .../docs/concepts/python-nodes/providers.mdx | 12 ++++++ .../concepts/python-nodes/sql-references.mdx | 4 ++ .../docs/docs/concepts/python-nodes/tasks.mdx | 7 ++++ .../rules/configuration-and-selection.mdx | 5 +++ .../content/docs/docs/concepts/scenarios.mdx | 2 +- .../src/content/docs/docs/concepts/seeds.mdx | 5 ++- .../content/docs/docs/concepts/snapshots.mdx | 9 +++++ .../content/docs/docs/concepts/sources.mdx | 8 +++- .../docs/integrations/dagster-reference.mdx | 1 + .../content/docs/docs/integrations/dlt.mdx | 6 +++ .../docs/docs/integrations/ingestr.mdx | 3 ++ 72 files changed, 394 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ff7ef67836..3dc9d8eb68 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ A model is a SQL file with a `MODEL()` header and a `SELECT`: ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (audits [not_null, unique]), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 5894fca18c..fa13f57633 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -17,6 +17,9 @@ plan, and ships tools for proving a change is correct. Use those tools; do not s - Macros are plain Python functions called as `@macro_name("arg")`. Constants and enums are `@const("name")` and `@enum("name").MEMBER`. - Config, column schemas, contracts and audits live in the `MODEL()` header, not YAML. +- Every model, scenario, seed, source, function, named hook, Python node and provider needs a + non-empty description (`MODEL (description "...")`, `description:` in YAML, a docstring or + `description=` in Python). A missing one is a `P010` compile error that shows the exact line to add. - `sqb compile` is offline: syntax, per-input binding, grouping semantics, contracts, metadata, column lineage and Rules run before warehouse execution. Explicit projections close output names even over open inputs. DuckDB/MotherDuck, PostgreSQL, Snowflake, and BigQuery enable expression @@ -36,6 +39,7 @@ plan, and ships tools for proving a change is correct. Use those tools; do not s ```sql -- models/marts/fact_orders.sql MODEL ( + description "One row per order with its payment total", materialized table, columns ( order_id (audits [not_null, unique]), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 8af247c1a8..56243ac9ac 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -47,6 +47,14 @@ Configure the maximum physical line width in `sqlbuild_project.toml` (the defaul line_width = 100 ``` +`MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; +`sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it +before compile does (see [Descriptions](../concepts/models/configuration.md#descriptions)). A model +described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +Format does not render templates, so a templated description that renders empty passes format and +is reported by `sqb compile`. +Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. + Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks are normalized as spaces, while blank lines preserve paragraph boundaries. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md index 992fffc89f..762d5b6339 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md @@ -150,7 +150,7 @@ def formatted_order_total(expression: str) -> str: ```sql -- models/commerce/orders.sql -MODEL(); +MODEL(description "Orders"); SELECT @formatted_order_total("subtotal") AS total ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index f48620c5a6..833b3dbd9e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -67,6 +67,7 @@ Attach threshold and sample policy where the audit is used: ```sql MODEL ( + description "Valid order rate", audits [ valid_order_rate ( condition "order_id IS NOT NULL", @@ -130,6 +131,7 @@ Attach them in the `MODEL()` header like any generic audit: ```sql MODEL ( + description "One row per order", materialized view, tags [staging], columns ( @@ -212,6 +214,7 @@ that is not visible from there is a compile error. See ```sql MODEL ( + description "One row per order", materialized table, audits [ expression_is_true ( @@ -320,6 +323,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw__orders + description: Orders columns: - name: id audits: @@ -344,6 +348,7 @@ Seeds use the same YAML audit syntax as sources, at table and column level: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -397,6 +402,7 @@ Audits on incremental models can run at different lifecycle phases: ```sql MODEL ( + description "One row per order", materialized incremental, ... columns ( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md index e2e1624208..9ab89d1670 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md @@ -31,6 +31,7 @@ SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")` ```sql MODEL ( + description "Finance orders built on the dbt fact table", tags [finance], columns (order_id (audits [not_null])), ); @@ -43,7 +44,7 @@ This resolves to the qualified warehouse table name from the dbt manifest (e.g. SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: ```sql -MODEL (tags [marts]); +MODEL (description "Mart view of downstream orders", tags [marts]); SELECT order_id FROM __ref("downstream_orders") ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md index 094d42f7ee..dd7aec6a84 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md @@ -93,6 +93,7 @@ inherited sample and request exhaustive comparison with `row_diff_sample_rows 0` ```sql MODEL ( + description "One row per order", materialized table, unique_key [order_id, line_id], row_diff_sample_rows 0, @@ -157,6 +158,7 @@ Numeric columns can have tolerance rules to avoid false positives from floating- ```sql MODEL ( + description "Example model", materialized incremental, ... row_diff_tolerances ( @@ -181,6 +183,7 @@ Columns that are expected to differ between the two sides (like timestamps or co ```sql MODEL ( + description "One row per order", materialized incremental, ... row_diff_exclude_columns [latest_order_status], diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md index 066c988fdf..46cd77daa9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md @@ -20,6 +20,7 @@ Use the enum name as the column type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md index 4043248d36..ef845f99e5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -24,11 +24,12 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `description`, `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line. `description` is required: a function without one fails compilation with `P010` (see [Descriptions](models/configuration.md#descriptions)): ```sql -- functions/sql/udf__is_completed_order.sql FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); @@ -50,9 +51,12 @@ from sqlbuild.functions import udf runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Whether an order status counts as completed.""" return order_status == "completed" ``` +The function docstring is its description; `@udf(description="...")` overrides it. One of them is required, or compilation fails with `P010`. The docstring is deployed with the function body, so editing it redeploys the function; a `description=` value does not. + Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. ### Using scalar UDFs @@ -75,6 +79,7 @@ Table functions return multiple rows and columns. They are written in SQL under ```sql -- functions/sql/table_fn__customer_orders.sql FUNCTION ( + description "Customer orders", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -165,6 +170,7 @@ The `@udf` decorator accepts these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| +| `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | | `returns` | Yes | SQL return type string | | `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 3bc4b8ee01..edc9f7eaab 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -28,6 +28,7 @@ Inserts new rows without modifying existing data. Optionally uses a cursor (read ```sql MODEL ( + description "One row per customer", materialized incremental, incremental_strategy append, cursor created_at, @@ -50,6 +51,7 @@ Deletes rows in the cursor range, then inserts the new delta. Requires either `c ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key [order_id], @@ -69,6 +71,7 @@ Upserts rows using a unique key. Matched rows are updated; unmatched rows are in ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -105,6 +108,7 @@ Use `full_refresh false` for models that must retain their normal cursor or micr ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, full_refresh false, @@ -144,6 +148,7 @@ When a model references multiple upstream inputs, `cursor_inputs` is required to ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -165,6 +170,7 @@ Cursor-based incremental models can read their effective interval with zero-argu ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -202,6 +208,7 @@ Lookback extends the start of the replay window backwards to re-process recent d ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, cursor event_date, @@ -227,6 +234,7 @@ Each batch has its own audit cycle: create delta, run delta audits, apply DML, a ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -265,6 +273,7 @@ microbatch_concurrency = true ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -284,6 +293,7 @@ Watermark microbatch models can declare what to do when their resolved range con ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -339,6 +349,7 @@ Alignment does two things: it floors the replay window edges to the coarsest gra ```sql -- Upstream: daily grain, 2d batches MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -353,6 +364,7 @@ MODEL ( -- Downstream: hourly grain, but aligns to day automatically MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -380,6 +392,7 @@ The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (s ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 978b7a6d77..7cf958daae 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -49,6 +49,7 @@ Use `@macro_name(...)` in a model query: ```sql MODEL ( + description "One row per order", materialized table, tags [marts], ); @@ -196,6 +197,7 @@ def grant_target(target: str) -> str: ```sql MODEL ( + description "Example model", post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], ); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md index b654019d97..ce3a77d0c4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md @@ -11,6 +11,7 @@ project-wide namespace. ```sql MODEL ( + description "One row per order", enums ( _state [OPEN, CLOSED], ), @@ -51,6 +52,7 @@ Use `constant(...)` when a private constant needs an exact type or rendering cho ```sql MODEL ( + description "Example model", constants ( _usd_rate constant( type decimal, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index eef9085a54..eff7b76918 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -27,6 +27,7 @@ Add `migrate_from` to the renamed column in the model header, naming the old col ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key order_id, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 3dca51e3a8..bde6b83c91 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -9,6 +9,7 @@ Online: https://sqlbuild.com/docs/concepts/models/configuration/ ## Contents - Common fields +- Descriptions - SQL analysis - Table fields - Incremental fields @@ -35,7 +36,7 @@ which accepts any keys. |-------|-------------| | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | -| `description` | Human-readable model description | +| `description` | Required human-readable model description; see [Descriptions](#descriptions) | | `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](column-migrations.md)). On a table or view model that declares its own `migrate_from`, a column `migrate_from` only names the old column in the [compatibility view](migrations.md#old-names) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | @@ -52,6 +53,43 @@ which accepts any keys. | `dynamic_columns` | Named pivot column families with `pivot_column`, `value_column`, `aggregate`, `type`, and `name_pattern` | | `audit_factories` | Audit factories that generate audit cases for this model; see [Audits](../audits.md) | +## Descriptions + +Every named resource must have a non-empty description. A missing description is a `P010` compile +error, so `sqb plan` and `sqb build` stop too. There is no setting to turn the check off. One +compile reports every missing description, each at the resource's declaration with the exact text +to add: + +```text +error[P010]: model 'orders_summary' has no description + --> models/marts/orders_summary.sql:1:1 + = help: to describe model 'orders_summary', add this to the MODEL header in models/marts/orders_summary.sql: + MODEL ( + description "What one row of orders_summary represents", + ... + ); +``` + +| Resource | Where the description goes | +|----------|----------------------------| +| Model | `MODEL (description "...")`, or the `description` of its `model_schema` | +| Scenario | `SCENARIO (description "...")` | +| Seed | `description:` on its entry in `seeds/*.yml` | +| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| SQL function or table function | `FUNCTION (description "...")` | +| Python UDF | `@udf(description="...")` or the function docstring | +| Named SQL hook | `HOOK (description "...")` | +| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | +| Provider | The provider class docstring | + +SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a +description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a +model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it +does). `sqb format --check` also fails when a `MODEL`, `SCENARIO`, `FUNCTION` or `HOOK` +header has no description (a `MODEL` header that uses `model_schema` is left to compile), and +`sqb format` still moves a leading comment into an empty model description. + ## SQL analysis SQL analysis has project, invocation, and model gates: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md index 755de65810..567a86e07a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md @@ -34,6 +34,7 @@ Under explicit mode, a model must declare `contract enforced` to activate shape ```sql MODEL ( + description "One row per order", materialized table, contract enforced, columns ( @@ -108,6 +109,7 @@ A column may use a declared enum as a portable logical domain type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md index 7fbfcd7bff..4f785a52d5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md @@ -89,6 +89,7 @@ Models may mix SQL and Python entries in one ordered hook list: ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md index 4aa0ad63fa..9441072c37 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -34,13 +34,14 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, channel="#data-builds"): + """Notify complete.""" ctx.log( f"Notify {channel}: " f"{ctx.model_name} completed during {ctx.phase}" ) ``` -By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks. +By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A hook needs a description: pass `description=` or give the function a docstring ([Descriptions](../configuration.md#descriptions)). A Python file may define multiple decorated hooks. ## Declared reads @@ -53,6 +54,7 @@ from sqlbuild.refs import model @hook(reads=model("dim_customers")) def refresh_lookup(ctx): + """Refresh lookup.""" customers = ctx.relation(model("dim_customers")) ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}") ``` @@ -90,6 +92,7 @@ Reference the hook by name and optionally pass keyword arguments: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ python( @@ -143,6 +146,7 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send( f"Model {ctx.model_name} built successfully" ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md index 753e1010d3..faf33a66bb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md @@ -38,7 +38,7 @@ GRANT SELECT ON @relation TO @role The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. -`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. +`HOOK()` accepts only a required, non-empty `description` ([Descriptions](../configuration.md#descriptions)). It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. @@ -50,6 +50,7 @@ Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` ```sql MODEL ( + description "Orders", materialized table, post_hooks [ sql( @@ -87,6 +88,7 @@ VALUES (@'relation', @'role') ```sql MODEL ( + description "Record access", post_hooks [ sql( "record_access", @@ -109,6 +111,7 @@ Use `inline_sql("...")` for SQL that is specific to one model: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), @@ -156,7 +159,7 @@ Context components can be combined into qualified identifiers without whitespace wrapper: ```sql -HOOK (); +HOOK (description "Create the reconstruct_book helper function"); CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() RETURNS INTEGER @@ -171,6 +174,7 @@ SQL hooks reference other resources the same way model SQL does, with `__ref(".. ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'), @@ -184,7 +188,7 @@ SELECT order_id FROM __ref("stg_orders") **`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`** ```sql -HOOK (); +HOOK (description "Record counts"); @count_rows(__ref("stg_orders")) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index b4470341b9..05ff176ab4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -13,7 +13,7 @@ The `materialized` field selects how a model becomes a warehouse relation. Creates or replaces a database view on each build: ```sql -MODEL (materialized view); +MODEL (description "One row per order", materialized view); SELECT id AS order_id, customer_id, status FROM __source("raw__orders") @@ -26,7 +26,7 @@ View audits run after the view has been replaced. A failing audit marks the buil By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. ```sql -MODEL (materialized table); +MODEL (description "One row per order", materialized table); SELECT customer_id, COUNT(*) AS total_orders FROM __ref("stg_orders") @@ -41,6 +41,7 @@ On a normal incremental run, applies append, delete/insert, or merge DML from a ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -60,6 +61,7 @@ Maintains historical row versions with SCD Type 2 semantics: ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -78,6 +80,7 @@ A project-local Python materialization can manage specialized persistence with a ```sql MODEL ( + description "One row per order", materialized partition_tracked, placeholders ( partition_start "'2026-04-01'", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index b65efc1bd3..3576e4d1c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -29,6 +29,7 @@ To declare a migration by hand, add `migrate_from` to the renamed model's header ```sql MODEL ( + description "One row per order", materialized incremental, migrate_from stg_orders, cursor order_date, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md index 348bbcfd67..02ac567e6a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md @@ -14,6 +14,7 @@ For a SQL model, declaring a type on any inline or reusable-schema column enable ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (type INTEGER), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md index 8140ad5614..34a2cf3d07 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -28,6 +28,7 @@ from sqlbuild.assets import asset, AssetContext @asset def orders_export(ctx: AssetContext): + """Orders export.""" path = write_orders_csv() return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) ``` @@ -41,6 +42,7 @@ Assets record whether they actually produced an artifact via `materialized`: ```python @asset def orders_export(ctx: AssetContext): + """Orders export.""" if nothing_changed(): return ctx.result(metadata={"status": "unchanged"}, materialized=False) return ctx.result(metadata={"path": export()}, materialized=True) @@ -61,6 +63,7 @@ from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_dashboard(ctx): + """Orders dashboard.""" result = ctx.result_of(export_orders) return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) ``` @@ -73,6 +76,7 @@ from sqlbuild.refs import model @asset(depends_on=model("fact_orders")) def orders_extract(ctx): + """Orders extract.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] return ctx.result(metadata={"rows": rows}, materialized=True) @@ -94,6 +98,7 @@ Assets can declare a schema for catalog and lineage purposes. This does not enfo }, ) def orders_export(ctx): + """Orders export.""" ... ``` @@ -112,6 +117,7 @@ from sqlbuild.retries import RetryPolicy @asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) def export(ctx): + """Export the order summary.""" if not ready(): return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT return ctx.result(metadata={"path": do_export()}, materialized=True) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 85a07e2094..eca32a9470 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -31,6 +31,7 @@ from python.tasks.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): + """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: return ctx.fail("no orders exported") @@ -47,6 +48,7 @@ from sqlbuild.refs import model @check(depends_on=model("fact_orders")) def orders_present(ctx): + """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] return ctx.pass_() if count else ctx.fail("no orders") @@ -61,6 +63,7 @@ Return a result through the context helpers, or a bool shorthand: ```python @check(depends_on=orders_asset) def rows_present(ctx): + """Rows present.""" return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail ``` @@ -81,6 +84,7 @@ Returning `None` is not allowed - checks must be explicit. ```python @check(depends_on=export_orders, severity="warn") def orders_freshness(ctx): + """Orders freshness.""" if stale(): return ctx.fail("export is stale") # recorded as a warning, does not fail the build return ctx.pass_() diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index f5c8a2e866..e0cdfed182 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -27,6 +27,7 @@ def export_assets(): for table in TABLES: @asset(name=f"export_{table}", tags=("export",)) def export(ctx, table=table): + """Export one table to object storage.""" return ctx.result(metadata={"table": table}, materialized=True) nodes.append(export) return nodes @@ -57,14 +58,17 @@ from sqlbuild.checks import check def orders_pipeline(): @loader(name="raw_orders") def load(ctx): + """Load orders from the storefront API.""" return fetch_orders() @asset(name="orders_export", depends_on=load) def export(ctx): + """Export the order summary.""" return ctx.result(materialized=True) @check(depends_on=export) def orders_export_check(ctx): + """Check that the orders export is ready.""" return ctx.pass_("export ready") return [load, export, orders_export_check] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index d3a6d8ac74..0cd4e2c50f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -43,6 +43,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customers from the CRM export.""" return [ {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, @@ -58,6 +59,7 @@ Declare a managed source in `sources/*.yml`. A managed source is bound to the lo ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -88,6 +90,7 @@ Full replace. The target is dropped and recreated from the loader output on ever ```yaml sources: - name: raw_countries + description: Countries managed: true write_strategy: table columns: @@ -104,6 +107,7 @@ Insert all returned rows into the target. No deduplication. ```yaml sources: - name: raw_webhook_events + description: Webhook events managed: true write_strategy: append columns: @@ -120,6 +124,7 @@ Delete rows in the cursor range, then insert replacements. Requires `cursor_colu ```yaml sources: - name: raw_order_events + description: Order events managed: true write_strategy: delete_insert cursor_column: event_at @@ -137,6 +142,7 @@ The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_colu ```python @loader def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" if ctx.current_cursor_value is None: return fetch_all_events() return fetch_events_since(ctx.current_cursor_value) @@ -149,6 +155,7 @@ Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: merge unique_key: customer_id @@ -171,6 +178,7 @@ If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader ```python @loader def raw_status(ctx: LoaderContext) -> None: + """Load the order status lookup.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " @@ -183,6 +191,7 @@ The source is still declared as managed, just without a `write_strategy`: ```yaml sources: - name: raw_status + description: Order status lookup managed: true columns: - name: status_id @@ -253,6 +262,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" return [ {"account_id": 1, "account_name": "Pawnee Parks"}, {"account_id": 2, "account_name": "Eagleton"}, @@ -260,6 +270,7 @@ def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: @loader(depends_on=[raw_accounts]) def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" accounts = ctx.loader(raw_accounts) rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") return [ @@ -277,6 +288,7 @@ Intermediate loaders (those referenced only via `depends_on`, with no managed so ```python @loader(destination="staging.shared_accounts") def raw_accounts(ctx: LoaderContext): + """Load customer accounts.""" ... ``` @@ -293,6 +305,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | `unique_key` | Column(s) used as the merge key (string or list of strings) | | `columns` | Column specifications with name, type, nullable, and description | | `contract` | `enforced` or `none` | +| `description` | What the loaded rows are; defaults to the function docstring. Required unless the source YAML entry has a `description` ([Descriptions](../models/configuration.md#descriptions)) | ## Auto-load during builds diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 95e5a8cef0..38cb8e3262 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -94,7 +94,7 @@ Every decorator accepts the same organizational metadata: | `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | -| `description` | Human-readable docs (defaults to the function docstring) | +| `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](../models/configuration.md#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | `@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. @@ -135,6 +135,7 @@ Tasks and assets return through `ctx.result(...)`: ```python @task def export_orders(ctx): + """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 1ae87b4d78..335a237f59 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -30,6 +30,7 @@ Create a Python file under `providers/` in your project. A provider is a class t from sqlbuild.providers import Provider class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str endpoint: str = "https://api.example.com" @@ -40,6 +41,8 @@ class WarehouseClient(Provider): self.session.close() ``` +The class docstring is the provider's description and is required; a provider without one fails compilation with `P010` ([Descriptions](../models/configuration.md#descriptions)). + `Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. ## Provider name @@ -53,6 +56,7 @@ Override the name explicitly with `provider_name`: ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" provider_name = "warehouse" api_key: str ``` @@ -68,6 +72,7 @@ from sqlbuild.tasks import task @task def export_orders(ctx, warehouse_client): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -81,6 +86,7 @@ from providers.warehouse_client import WarehouseClient @task def export_orders(ctx, warehouse_client: WarehouseClient): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -103,11 +109,13 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send(f"Model {ctx.model_name} built successfully") ``` ```sql MODEL ( + description "One row per order", materialized table, post_hooks [python("notify_complete")], ); @@ -118,6 +126,7 @@ Providers are also available on the `HookContext` via `ctx.providers`: ```python @hook def notify_complete(ctx): + """Notify complete.""" notifier = ctx.providers.slack_notifier notifier.send(f"Model {ctx.model_name} built successfully") ``` @@ -129,6 +138,7 @@ All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `Loader ```python @task def export_orders(ctx): + """Export orders.""" client = ctx.providers.warehouse_client client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -145,6 +155,7 @@ Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str def setup(self, ctx): @@ -164,6 +175,7 @@ Because `Provider` extends `pydantic-settings BaseSettings`, fields without defa ```python class SlackNotifier(Provider): + """Posts build notifications to a chat channel.""" slack_token: str # reads SLACK_TOKEN from environment channel: str = "#builds" # has a default, environment variable is optional ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 3ed1ebdd4c..54c656c706 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -26,14 +26,17 @@ from sqlbuild.tasks import task @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" ... @task(depends_on=source("raw_orders")) def inspect_raw(ctx): + """Inspect raw.""" ... @task(depends_on=seed("country_codes")) def export_countries(ctx): + """Export countries.""" ... ``` @@ -53,6 +56,7 @@ Always resolve a reference to its concrete relation with `ctx.relation(...)` ins ```python @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT * FROM {relation}").fetchall() return ctx.result(metadata={"rows": len(rows)}) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md index 032ca4c17b..73d67cd81d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md @@ -28,6 +28,7 @@ from sqlbuild.tasks import task, TaskContext @task def export_orders(ctx: TaskContext): + """Export orders.""" rows = fetch_orders() return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) ``` @@ -41,10 +42,12 @@ Tasks declare dependencies with `depends_on`, accepting a single function, a tup ```python @task def fetch_orders(ctx: TaskContext): + """Fetch orders.""" return ctx.result(payload=download_orders()) @task(depends_on=fetch_orders) def summarize_orders(ctx: TaskContext): + """Summarize orders.""" result = ctx.result_of(fetch_orders) return ctx.result(metadata={"count": len(result.payload)}) ``` @@ -58,6 +61,7 @@ Tasks may depend on other tasks, assets, and loaders. They may **not** depend on ```python @task def build_export(ctx: TaskContext): + """Build export.""" return ctx.result( payload={"path": "/exports/orders.csv"}, metadata={"rows": 1200}, @@ -75,6 +79,7 @@ Return `ctx.skip(...)` to skip a task: ```python @task def export_if_present(ctx: TaskContext): + """Export if present.""" if not new_files_available(): return ctx.skip("no new files") return ctx.result(payload=do_export()) @@ -90,6 +95,7 @@ from sqlbuild.tasks import task, SkipMode @task def optional_step(ctx): + """Optional step.""" return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" ``` @@ -103,6 +109,7 @@ from sqlbuild.tasks import task @task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) def call_api(ctx): + """Call api.""" return ctx.result(payload=fetch_from_flaky_api()) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 833d151170..03964b35c9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -41,6 +41,11 @@ XSQBR001 → XSQBR Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching semantics and takes precedence over `select`. +A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired +when descriptions became the compile error `P010` +([Descriptions](../models/configuration.md#descriptions)); remove it from `select` and +`ignore`. + Use the catalogue to inspect available codes and metadata: ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md index cbc47164d4..df22b9868d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -120,7 +120,7 @@ The header is metadata only: | Field | Description | |-------|-------------| -| `description` | Optional description string | +| `description` | Required description of the behaviour the scenario proves ([Descriptions](models/configuration.md#descriptions)) | | `tags` | Optional list of string tags | ## How scenarios execute diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md index 303b4ef1d8..34e3d3f4b1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md @@ -55,7 +55,7 @@ Seed YAML accepts only these keys; any other key is a compile error: | Seed | `name`, `description`, `database`, `schema`, `meta`, `tags`, `csv_settings`, `columns`, `audits` | | Column | `name`, `type`, `nullable`, `description`, `meta`, `audits` | -Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). +Every seed must have a YAML declaration with a non-empty `description` ([Descriptions](models/configuration.md#descriptions)) and at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). ## Referencing seeds @@ -89,6 +89,7 @@ Attach audits to a seed or its columns with the same syntax as source audits: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -111,6 +112,7 @@ Seeds inherit the project's default database and schema. You can override these ```yaml seeds: - name: waffle_types + description: Waffle types database: analytics schema: lookups columns: @@ -127,6 +129,7 @@ For non-standard CSV formats, configure parsing behavior with `csv_settings`: ```yaml seeds: - name: european_prices + description: European prices csv_settings: delimiter: ";" encoding: utf-8 diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md index 0d318ce936..b70d310a89 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md @@ -40,6 +40,7 @@ Use when your source has a reliable column recording when the entity changed. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -63,6 +64,7 @@ Use when the source does not have a reliable update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -91,6 +93,7 @@ Use for daily full exports or periodic snapshots without a business update times ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -114,6 +117,7 @@ Use for historical observations that include a business update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -139,6 +143,7 @@ Use for CDC tables, audit logs, or historical backfills where rows are individua ```sql MODEL ( + description "One row per order", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -244,6 +249,7 @@ The model `snapshot_full_refresh` field can only make the policy **stricter** th ```sql MODEL ( + description "Example model", materialized snapshot, ... snapshot_full_refresh deny, @@ -266,6 +272,7 @@ Snapshot models support the same audit system as other materializations. Audits ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -340,6 +347,7 @@ JOIN customer_snapshot c ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [user_id, role_id], snapshot_strategy check, @@ -357,6 +365,7 @@ FROM __source("user_role_daily") ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [product_id], snapshot_strategy timestamp, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md index 32e33b0b15..9d087b5068 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -25,6 +25,7 @@ Point at an existing table or view in your warehouse: ```yaml sources: - name: raw_events + description: Events database: analytics schema: raw table: events @@ -39,6 +40,7 @@ Define source data inline as a SQL expression. No external tables or setup scrip ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT * FROM (VALUES (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), @@ -59,6 +61,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw_orders + description: Orders columns: - name: id audits: @@ -82,6 +85,7 @@ Type enforcement is implicit for sources. If any column declares a `type`, SQLBu ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name columns: @@ -104,6 +108,7 @@ Sources can be loaded by Python functions instead of pointing at existing tables ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -135,6 +140,7 @@ Configure freshness per source with a `freshness:` block: ```yaml sources: - name: raw_events + description: Events schema: raw table: events freshness: @@ -245,7 +251,7 @@ Use [`sqb freshness`](../cli/freshness.md) to observe source freshness on demand | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | | `unique_key` | Merge key column(s) (required for `merge`) | | `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | -| `description` | Human-readable description | +| `description` | Required human-readable description ([Descriptions](models/configuration.md#descriptions)) | | `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | | `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | | `columns` | Column declarations with `name`, `type`, `nullable`, `description`, `meta`, and `audits` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md index ca5204b7bd..0fbe0b5d99 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md @@ -219,6 +219,7 @@ A source can provide a stable Dagster-native key without replacing the translato ```yaml sources: - name: raw__orders + description: Orders meta: dagster: asset_key: [warehouse, raw, orders] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 71cbda289c..851ae4f9cc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -49,6 +49,7 @@ dlt_sources: type: header_link resources: - name: raw_github_issues + description: Issues from the GitHub API write_disposition: append endpoint: path: "repos/${github_owner}/${github_repo}/issues" @@ -69,6 +70,7 @@ dlt_sources: credentials: "${postgres_connection_string}" resources: - name: raw_customers + description: Customers from the CRM database table: customers write_disposition: merge primary_key: id @@ -86,6 +88,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + description: Events from the event export files reader: csv write_disposition: append ``` @@ -97,6 +100,7 @@ dlt_sources: | Key | Description | |-----|-------------| | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | +| `description` | What the loaded rows are. Required ([Descriptions](../concepts/models/configuration.md#descriptions)). | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | @@ -148,6 +152,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_github_issues(ctx: LoaderContext): + """Github issues.""" source = rest_api_source({ "client": { "base_url": "https://api.github.com/", @@ -182,6 +187,7 @@ Bind it to a source with `managed: true` in `sources/*.yml`: # sources/github.yml sources: - name: raw_github_issues + description: Github issues managed: true table: issues columns: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index 9621233a2c..b573813aed 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -48,6 +48,7 @@ Replicate a table from PostgreSQL into your local DuckDB project: # sources/raw.yml sources: - name: raw_orders + description: Orders table: orders ingestr: source_uri: "postgresql://user:pass@host:5432/mydb" @@ -63,6 +64,7 @@ Load Stripe charges with incremental merge on a primary key: ```yaml sources: - name: raw_stripe_charges + description: Stripe charges table: charges ingestr: source_uri: "stripe://${stripe_api_key}" @@ -105,6 +107,7 @@ All string fields in the `ingestr` block support SQLBuild project variable subst ```yaml sources: - name: raw_orders + description: Orders ingestr: source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" source_table: "public.orders" diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index bafe69f9e4..b9417cff64 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -39,6 +39,14 @@ Configure the maximum physical line width in `sqlbuild_project.toml` (the defaul line_width = 100 ``` +`MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; +`sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it +before compile does (see [Descriptions](/docs/concepts/models/configuration#descriptions)). A model +described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +Format does not render templates, so a templated description that renders empty passes format and +is reported by `sqb compile`. +Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. + Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks are normalized as spaces, while blank lines preserve paragraph boundaries. diff --git a/website/src/content/docs/docs/cli/scope.mdx b/website/src/content/docs/docs/cli/scope.mdx index 584ba79b4a..95f626b6e4 100644 --- a/website/src/content/docs/docs/cli/scope.mdx +++ b/website/src/content/docs/docs/cli/scope.mdx @@ -132,7 +132,7 @@ def formatted_order_total(expression: str) -> str: ```sql -- models/commerce/orders.sql -MODEL(); +MODEL(description "Orders"); SELECT @formatted_order_total("subtotal") AS total ``` diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 93ff76ab90..9b2cad7b76 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -48,6 +48,7 @@ Attach threshold and sample policy where the audit is used: ```sql MODEL ( + description "Valid order rate", audits [ valid_order_rate ( condition "order_id IS NOT NULL", @@ -112,6 +113,7 @@ Attach them in the `MODEL()` header like any generic audit: ```sql MODEL ( + description "One row per order", materialized view, tags [staging], columns ( @@ -194,6 +196,7 @@ that is not visible from there is a compile error. See ```sql MODEL ( + description "One row per order", materialized table, audits [ expression_is_true ( @@ -302,6 +305,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw__orders + description: Orders columns: - name: id audits: @@ -326,6 +330,7 @@ Seeds use the same YAML audit syntax as sources, at table and column level: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -379,6 +384,7 @@ Audits on incremental models can run at different lifecycle phases: ```sql MODEL ( + description "One row per order", materialized incremental, ... columns ( diff --git a/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx b/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx index 32f9ff8f05..74e89a373e 100644 --- a/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx +++ b/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx @@ -28,6 +28,7 @@ SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")` ```sql MODEL ( + description "Finance orders built on the dbt fact table", tags [finance], columns (order_id (audits [not_null])), ); @@ -40,7 +41,7 @@ This resolves to the qualified warehouse table name from the dbt manifest (e.g. SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: ```sql -MODEL (tags [marts]); +MODEL (description "Mart view of downstream orders", tags [marts]); SELECT order_id FROM __ref("downstream_orders") ``` diff --git a/website/src/content/docs/docs/concepts/diff.mdx b/website/src/content/docs/docs/concepts/diff.mdx index 3e435a4246..6805e5e852 100644 --- a/website/src/content/docs/docs/concepts/diff.mdx +++ b/website/src/content/docs/docs/concepts/diff.mdx @@ -81,6 +81,7 @@ inherited sample and request exhaustive comparison with `row_diff_sample_rows 0` ```sql MODEL ( + description "One row per order", materialized table, unique_key [order_id, line_id], row_diff_sample_rows 0, @@ -145,6 +146,7 @@ Numeric columns can have tolerance rules to avoid false positives from floating- ```sql MODEL ( + description "Example model", materialized incremental, ... row_diff_tolerances ( @@ -169,6 +171,7 @@ Columns that are expected to differ between the two sides (like timestamps or co ```sql MODEL ( + description "One row per order", materialized incremental, ... row_diff_exclude_columns [latest_order_status], diff --git a/website/src/content/docs/docs/concepts/enums/model-contracts.mdx b/website/src/content/docs/docs/concepts/enums/model-contracts.mdx index e9a1a579d4..635a51819a 100644 --- a/website/src/content/docs/docs/concepts/enums/model-contracts.mdx +++ b/website/src/content/docs/docs/concepts/enums/model-contracts.mdx @@ -17,6 +17,7 @@ Use the enum name as the column type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/website/src/content/docs/docs/concepts/functions.mdx b/website/src/content/docs/docs/concepts/functions.mdx index 56f9f86665..07df454f94 100644 --- a/website/src/content/docs/docs/concepts/functions.mdx +++ b/website/src/content/docs/docs/concepts/functions.mdx @@ -11,11 +11,12 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `description`, `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line. `description` is required: a function without one fails compilation with `P010` (see [Descriptions](/docs/concepts/models/configuration#descriptions)): ```sql -- functions/sql/udf__is_completed_order.sql FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); @@ -37,9 +38,12 @@ from sqlbuild.functions import udf runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Whether an order status counts as completed.""" return order_status == "completed" ``` +The function docstring is its description; `@udf(description="...")` overrides it. One of them is required, or compilation fails with `P010`. The docstring is deployed with the function body, so editing it redeploys the function; a `description=` value does not. + Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. ### Using scalar UDFs @@ -62,6 +66,7 @@ Table functions return multiple rows and columns. They are written in SQL under ```sql -- functions/sql/table_fn__customer_orders.sql FUNCTION ( + description "Customer orders", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -152,6 +157,7 @@ The `@udf` decorator accepts these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| +| `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | | `returns` | Yes | SQL return type string | | `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 8858022557..f98ac33f7b 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -17,6 +17,7 @@ Inserts new rows without modifying existing data. Optionally uses a cursor (read ```sql MODEL ( + description "One row per customer", materialized incremental, incremental_strategy append, cursor created_at, @@ -39,6 +40,7 @@ Deletes rows in the cursor range, then inserts the new delta. Requires either `c ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key [order_id], @@ -58,6 +60,7 @@ Upserts rows using a unique key. Matched rows are updated; unmatched rows are in ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -94,6 +97,7 @@ Use `full_refresh false` for models that must retain their normal cursor or micr ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, full_refresh false, @@ -133,6 +137,7 @@ When a model references multiple upstream inputs, `cursor_inputs` is required to ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -154,6 +159,7 @@ Cursor-based incremental models can read their effective interval with zero-argu ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -191,6 +197,7 @@ Lookback extends the start of the replay window backwards to re-process recent d ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, cursor event_date, @@ -216,6 +223,7 @@ Each batch has its own audit cycle: create delta, run delta audits, apply DML, a ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -254,6 +262,7 @@ microbatch_concurrency = true ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -273,6 +282,7 @@ Watermark microbatch models can declare what to do when their resolved range con ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -328,6 +338,7 @@ Alignment does two things: it floors the replay window edges to the coarsest gra ```sql -- Upstream: daily grain, 2d batches MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -342,6 +353,7 @@ MODEL ( -- Downstream: hourly grain, but aligns to day automatically MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -369,6 +381,7 @@ The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (s ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 57ee084ffc..9eccc3bc64 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -37,6 +37,7 @@ Use `@macro_name(...)` in a model query: ```sql MODEL ( + description "One row per order", materialized table, tags [marts], ); @@ -187,6 +188,7 @@ def grant_target(target: str) -> str: ```sql MODEL ( + description "Example model", post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], ); diff --git a/website/src/content/docs/docs/concepts/model-private-values.mdx b/website/src/content/docs/docs/concepts/model-private-values.mdx index 886f2a1670..baa5733254 100644 --- a/website/src/content/docs/docs/concepts/model-private-values.mdx +++ b/website/src/content/docs/docs/concepts/model-private-values.mdx @@ -8,6 +8,7 @@ project-wide namespace. ```sql MODEL ( + description "One row per order", enums ( _state [OPEN, CLOSED], ), @@ -48,6 +49,7 @@ Use `constant(...)` when a private constant needs an exact type or rendering cho ```sql MODEL ( + description "Example model", constants ( _usd_rate constant( type decimal, diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 4fb598e3a2..05408005d0 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -15,6 +15,7 @@ Add `migrate_from` to the renamed column in the model header, naming the old col ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key order_id, diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 010e0a66d8..248f29979a 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -22,7 +22,7 @@ which accepts any keys. |-------|-------------| | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | -| `description` | Human-readable model description | +| `description` | Required human-readable model description; see [Descriptions](#descriptions) | | `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](/docs/concepts/models/column-migrations)). On a table or view model that declares its own `migrate_from`, a column `migrate_from` only names the old column in the [compatibility view](/docs/concepts/models/migrations#old-names) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | @@ -39,6 +39,43 @@ which accepts any keys. | `dynamic_columns` | Named pivot column families with `pivot_column`, `value_column`, `aggregate`, `type`, and `name_pattern` | | `audit_factories` | Audit factories that generate audit cases for this model; see [Audits](/docs/concepts/audits) | +## Descriptions + +Every named resource must have a non-empty description. A missing description is a `P010` compile +error, so `sqb plan` and `sqb build` stop too. There is no setting to turn the check off. One +compile reports every missing description, each at the resource's declaration with the exact text +to add: + +```text +error[P010]: model 'orders_summary' has no description + --> models/marts/orders_summary.sql:1:1 + = help: to describe model 'orders_summary', add this to the MODEL header in models/marts/orders_summary.sql: + MODEL ( + description "What one row of orders_summary represents", + ... + ); +``` + +| Resource | Where the description goes | +|----------|----------------------------| +| Model | `MODEL (description "...")`, or the `description` of its `model_schema` | +| Scenario | `SCENARIO (description "...")` | +| Seed | `description:` on its entry in `seeds/*.yml` | +| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| SQL function or table function | `FUNCTION (description "...")` | +| Python UDF | `@udf(description="...")` or the function docstring | +| Named SQL hook | `HOOK (description "...")` | +| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | +| Provider | The provider class docstring | + +SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a +description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a +model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it +does). `sqb format --check` also fails when a `MODEL`, `SCENARIO`, `FUNCTION` or `HOOK` +header has no description (a `MODEL` header that uses `model_schema` is left to compile), and +`sqb format` still moves a leading comment into an empty model description. + ## SQL analysis SQL analysis has project, invocation, and model gates: diff --git a/website/src/content/docs/docs/concepts/models/contracts.mdx b/website/src/content/docs/docs/concepts/models/contracts.mdx index 9a738a7620..aefe17679d 100644 --- a/website/src/content/docs/docs/concepts/models/contracts.mdx +++ b/website/src/content/docs/docs/concepts/models/contracts.mdx @@ -22,6 +22,7 @@ Under explicit mode, a model must declare `contract enforced` to activate shape ```sql MODEL ( + description "One row per order", materialized table, contract enforced, columns ( @@ -96,6 +97,7 @@ A column may use a declared enum as a portable logical domain type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/website/src/content/docs/docs/concepts/models/hooks.mdx b/website/src/content/docs/docs/concepts/models/hooks.mdx index 1f52be6b81..575a0e3710 100644 --- a/website/src/content/docs/docs/concepts/models/hooks.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks.mdx @@ -77,6 +77,7 @@ Models may mix SQL and Python entries in one ordered hook list: ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx index 872516e359..c92be18ff3 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx @@ -21,13 +21,14 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, channel="#data-builds"): + """Notify complete.""" ctx.log( f"Notify {channel}: " f"{ctx.model_name} completed during {ctx.phase}" ) ``` -By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks. +By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A hook needs a description: pass `description=` or give the function a docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)). A Python file may define multiple decorated hooks. ## Declared reads @@ -41,6 +42,7 @@ from sqlbuild.refs import model @hook(reads=model("dim_customers")) def refresh_lookup(ctx): + """Refresh lookup.""" customers = ctx.relation(model("dim_customers")) ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}") ``` @@ -78,6 +80,7 @@ Reference the hook by name and optionally pass keyword arguments: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ python( @@ -132,6 +135,7 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send( f"Model {ctx.model_name} built successfully" ) diff --git a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx index f134a538c5..87e1eb7ed9 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx @@ -25,7 +25,7 @@ GRANT SELECT ON @relation TO @role The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. -`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. +`HOOK()` accepts only a required, non-empty `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)). It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. @@ -37,6 +37,7 @@ Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` ```sql MODEL ( + description "Orders", materialized table, post_hooks [ sql( @@ -74,6 +75,7 @@ VALUES (@'relation', @'role') ```sql MODEL ( + description "Record access", post_hooks [ sql( "record_access", @@ -96,6 +98,7 @@ Use `inline_sql("...")` for SQL that is specific to one model: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), @@ -143,7 +146,7 @@ Context components can be combined into qualified identifiers without whitespace wrapper: ```sql -HOOK (); +HOOK (description "Create the reconstruct_book helper function"); CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() RETURNS INTEGER @@ -158,6 +161,7 @@ SQL hooks reference other resources the same way model SQL does, with `__ref(".. ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'), @@ -171,7 +175,7 @@ SELECT order_id FROM __ref("stg_orders") **`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`** ```sql -HOOK (); +HOOK (description "Record counts"); @count_rows(__ref("stg_orders")) ``` diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 397857d106..22d0aae228 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -10,7 +10,7 @@ The `materialized` field selects how a model becomes a warehouse relation. Creates or replaces a database view on each build: ```sql -MODEL (materialized view); +MODEL (description "One row per order", materialized view); SELECT id AS order_id, customer_id, status FROM __source("raw__orders") @@ -23,7 +23,7 @@ View audits run after the view has been replaced. A failing audit marks the buil By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. ```sql -MODEL (materialized table); +MODEL (description "One row per order", materialized table); SELECT customer_id, COUNT(*) AS total_orders FROM __ref("stg_orders") @@ -38,6 +38,7 @@ On a normal incremental run, applies append, delete/insert, or merge DML from a ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -57,6 +58,7 @@ Maintains historical row versions with SCD Type 2 semantics: ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -75,6 +77,7 @@ A project-local Python materialization can manage specialized persistence with a ```sql MODEL ( + description "One row per order", materialized partition_tracked, placeholders ( partition_start "'2026-04-01'", diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index b142788471..4f2d9af630 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -15,6 +15,7 @@ To declare a migration by hand, add `migrate_from` to the renamed model's header ```sql MODEL ( + description "One row per order", materialized incremental, migrate_from stg_orders, cursor order_date, diff --git a/website/src/content/docs/docs/concepts/models/type-enforcement.mdx b/website/src/content/docs/docs/concepts/models/type-enforcement.mdx index 7bcbeea1f0..65fc5c323a 100644 --- a/website/src/content/docs/docs/concepts/models/type-enforcement.mdx +++ b/website/src/content/docs/docs/concepts/models/type-enforcement.mdx @@ -11,6 +11,7 @@ For a SQL model, declaring a type on any inline or reusable-schema column enable ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (type INTEGER), diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx index 31f0c61a10..50fcbb7b43 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx @@ -16,6 +16,7 @@ from sqlbuild.assets import asset, AssetContext @asset def orders_export(ctx: AssetContext): + """Orders export.""" path = write_orders_csv() return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) ``` @@ -29,6 +30,7 @@ Assets record whether they actually produced an artifact via `materialized`: ```python @asset def orders_export(ctx: AssetContext): + """Orders export.""" if nothing_changed(): return ctx.result(metadata={"status": "unchanged"}, materialized=False) return ctx.result(metadata={"path": export()}, materialized=True) @@ -50,6 +52,7 @@ from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_dashboard(ctx): + """Orders dashboard.""" result = ctx.result_of(export_orders) return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) ``` @@ -63,6 +66,7 @@ from sqlbuild.refs import model @asset(depends_on=model("fact_orders")) def orders_extract(ctx): + """Orders extract.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] return ctx.result(metadata={"rows": rows}, materialized=True) @@ -84,6 +88,7 @@ Assets can declare a schema for catalog and lineage purposes. This does not enfo }, ) def orders_export(ctx): + """Orders export.""" ... ``` @@ -103,6 +108,7 @@ from sqlbuild.retries import RetryPolicy @asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) def export(ctx): + """Export the order summary.""" if not ready(): return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT return ctx.result(metadata={"path": do_export()}, materialized=True) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index 668acfd5f3..c4d5355e9e 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -19,6 +19,7 @@ from python.tasks.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): + """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: return ctx.fail("no orders exported") @@ -36,6 +37,7 @@ from sqlbuild.refs import model @check(depends_on=model("fact_orders")) def orders_present(ctx): + """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] return ctx.pass_() if count else ctx.fail("no orders") @@ -50,6 +52,7 @@ Return a result through the context helpers, or a bool shorthand: ```python @check(depends_on=orders_asset) def rows_present(ctx): + """Rows present.""" return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail ``` @@ -70,6 +73,7 @@ Returning `None` is not allowed - checks must be explicit. ```python @check(depends_on=export_orders, severity="warn") def orders_freshness(ctx): + """Orders freshness.""" if stale(): return ctx.fail("export is stale") # recorded as a warning, does not fail the build return ctx.pass_() diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index c2af78862c..6994e8fbee 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -25,6 +25,7 @@ def export_assets(): for table in TABLES: @asset(name=f"export_{table}", tags=("export",)) def export(ctx, table=table): + """Export one table to object storage.""" return ctx.result(metadata={"table": table}, materialized=True) nodes.append(export) return nodes @@ -56,14 +57,17 @@ from sqlbuild.checks import check def orders_pipeline(): @loader(name="raw_orders") def load(ctx): + """Load orders from the storefront API.""" return fetch_orders() @asset(name="orders_export", depends_on=load) def export(ctx): + """Export the order summary.""" return ctx.result(materialized=True) @check(depends_on=export) def orders_export_check(ctx): + """Check that the orders export is ready.""" return ctx.pass_("export ready") return [load, export, orders_export_check] diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 2c1e44c548..8f8da8d450 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -24,6 +24,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customers from the CRM export.""" return [ {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, @@ -39,6 +40,7 @@ Declare a managed source in `sources/*.yml`. A managed source is bound to the lo ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -69,6 +71,7 @@ Full replace. The target is dropped and recreated from the loader output on ever ```yaml sources: - name: raw_countries + description: Countries managed: true write_strategy: table columns: @@ -85,6 +88,7 @@ Insert all returned rows into the target. No deduplication. ```yaml sources: - name: raw_webhook_events + description: Webhook events managed: true write_strategy: append columns: @@ -101,6 +105,7 @@ Delete rows in the cursor range, then insert replacements. Requires `cursor_colu ```yaml sources: - name: raw_order_events + description: Order events managed: true write_strategy: delete_insert cursor_column: event_at @@ -118,6 +123,7 @@ The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_colu ```python @loader def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" if ctx.current_cursor_value is None: return fetch_all_events() return fetch_events_since(ctx.current_cursor_value) @@ -130,6 +136,7 @@ Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: merge unique_key: customer_id @@ -152,6 +159,7 @@ If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader ```python @loader def raw_status(ctx: LoaderContext) -> None: + """Load the order status lookup.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " @@ -164,6 +172,7 @@ The source is still declared as managed, just without a `write_strategy`: ```yaml sources: - name: raw_status + description: Order status lookup managed: true columns: - name: status_id @@ -234,6 +243,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" return [ {"account_id": 1, "account_name": "Pawnee Parks"}, {"account_id": 2, "account_name": "Eagleton"}, @@ -241,6 +251,7 @@ def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: @loader(depends_on=[raw_accounts]) def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" accounts = ctx.loader(raw_accounts) rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") return [ @@ -258,6 +269,7 @@ Intermediate loaders (those referenced only via `depends_on`, with no managed so ```python @loader(destination="staging.shared_accounts") def raw_accounts(ctx: LoaderContext): + """Load customer accounts.""" ... ``` @@ -274,6 +286,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | `unique_key` | Column(s) used as the merge key (string or list of strings) | | `columns` | Column specifications with name, type, nullable, and description | | `contract` | `enforced` or `none` | +| `description` | What the loaded rows are; defaults to the function docstring. Required unless the source YAML entry has a `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)) | ## Auto-load during builds diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 9ac5a63176..5087778955 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -78,7 +78,7 @@ Every decorator accepts the same organizational metadata: | `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | -| `description` | Human-readable docs (defaults to the function docstring) | +| `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | `@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. @@ -119,6 +119,7 @@ Tasks and assets return through `ctx.result(...)`: ```python @task def export_orders(ctx): + """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 1019f76824..2eb9b35d0c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -14,6 +14,7 @@ Create a Python file under `providers/` in your project. A provider is a class t from sqlbuild.providers import Provider class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str endpoint: str = "https://api.example.com" @@ -24,6 +25,8 @@ class WarehouseClient(Provider): self.session.close() ``` +The class docstring is the provider's description and is required; a provider without one fails compilation with `P010` ([Descriptions](/docs/concepts/models/configuration#descriptions)). + `Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. ## Provider name @@ -37,6 +40,7 @@ Override the name explicitly with `provider_name`: ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" provider_name = "warehouse" api_key: str ``` @@ -52,6 +56,7 @@ from sqlbuild.tasks import task @task def export_orders(ctx, warehouse_client): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -65,6 +70,7 @@ from providers.warehouse_client import WarehouseClient @task def export_orders(ctx, warehouse_client: WarehouseClient): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -87,11 +93,13 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send(f"Model {ctx.model_name} built successfully") ``` ```sql MODEL ( + description "One row per order", materialized table, post_hooks [python("notify_complete")], ); @@ -102,6 +110,7 @@ Providers are also available on the `HookContext` via `ctx.providers`: ```python @hook def notify_complete(ctx): + """Notify complete.""" notifier = ctx.providers.slack_notifier notifier.send(f"Model {ctx.model_name} built successfully") ``` @@ -113,6 +122,7 @@ All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `Loader ```python @task def export_orders(ctx): + """Export orders.""" client = ctx.providers.warehouse_client client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -129,6 +139,7 @@ Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str def setup(self, ctx): @@ -148,6 +159,7 @@ Because `Provider` extends `pydantic-settings BaseSettings`, fields without defa ```python class SlackNotifier(Provider): + """Posts build notifications to a chat channel.""" slack_token: str # reads SLACK_TOKEN from environment channel: str = "#builds" # has a default, environment variable is optional ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx index 7c31978f8e..e0101e9095 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx @@ -18,16 +18,19 @@ from sqlbuild.tasks import task @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" ... @task(depends_on=source("raw_orders")) def inspect_raw(ctx): + """Inspect raw.""" ... @task(depends_on=seed("country_codes")) def export_countries(ctx): + """Export countries.""" ... ``` @@ -47,6 +50,7 @@ Always resolve a reference to its concrete relation with `ctx.relation(...)` ins ```python @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT * FROM {relation}").fetchall() return ctx.result(metadata={"rows": len(rows)}) diff --git a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx index 836879bdfc..f7f4c4860a 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx @@ -16,6 +16,7 @@ from sqlbuild.tasks import task, TaskContext @task def export_orders(ctx: TaskContext): + """Export orders.""" rows = fetch_orders() return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) ``` @@ -29,11 +30,13 @@ Tasks declare dependencies with `depends_on`, accepting a single function, a tup ```python @task def fetch_orders(ctx: TaskContext): + """Fetch orders.""" return ctx.result(payload=download_orders()) @task(depends_on=fetch_orders) def summarize_orders(ctx: TaskContext): + """Summarize orders.""" result = ctx.result_of(fetch_orders) return ctx.result(metadata={"count": len(result.payload)}) ``` @@ -47,6 +50,7 @@ Tasks may depend on other tasks, assets, and loaders. They may **not** depend on ```python @task def build_export(ctx: TaskContext): + """Build export.""" return ctx.result( payload={"path": "/exports/orders.csv"}, metadata={"rows": 1200}, @@ -64,6 +68,7 @@ Return `ctx.skip(...)` to skip a task: ```python @task def export_if_present(ctx: TaskContext): + """Export if present.""" if not new_files_available(): return ctx.skip("no new files") return ctx.result(payload=do_export()) @@ -80,6 +85,7 @@ from sqlbuild.tasks import task, SkipMode @task def optional_step(ctx): + """Optional step.""" return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" ``` @@ -94,6 +100,7 @@ from sqlbuild.tasks import task @task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) def call_api(ctx): + """Call api.""" return ctx.result(payload=fetch_from_flaky_api()) ``` diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 44e414a286..c38af35958 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -32,6 +32,11 @@ XSQBR001 → XSQBR Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching semantics and takes precedence over `select`. +A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired +when descriptions became the compile error `P010` +([Descriptions](/docs/concepts/models/configuration#descriptions)); remove it from `select` and +`ignore`. + Use the catalogue to inspect available codes and metadata: ```bash diff --git a/website/src/content/docs/docs/concepts/scenarios.mdx b/website/src/content/docs/docs/concepts/scenarios.mdx index b7dd4bbfeb..cd6934f6eb 100644 --- a/website/src/content/docs/docs/concepts/scenarios.mdx +++ b/website/src/content/docs/docs/concepts/scenarios.mdx @@ -108,7 +108,7 @@ The header is metadata only: | Field | Description | |-------|-------------| -| `description` | Optional description string | +| `description` | Required description of the behaviour the scenario proves ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `tags` | Optional list of string tags | ## How scenarios execute diff --git a/website/src/content/docs/docs/concepts/seeds.mdx b/website/src/content/docs/docs/concepts/seeds.mdx index 05d864b6d1..cfbe20b4b9 100644 --- a/website/src/content/docs/docs/concepts/seeds.mdx +++ b/website/src/content/docs/docs/concepts/seeds.mdx @@ -43,7 +43,7 @@ Seed YAML accepts only these keys; any other key is a compile error: | Seed | `name`, `description`, `database`, `schema`, `meta`, `tags`, `csv_settings`, `columns`, `audits` | | Column | `name`, `type`, `nullable`, `description`, `meta`, `audits` | -Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). +Every seed must have a YAML declaration with a non-empty `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)) and at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). ## Referencing seeds @@ -77,6 +77,7 @@ Attach audits to a seed or its columns with the same syntax as source audits: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -99,6 +100,7 @@ Seeds inherit the project's default database and schema. You can override these ```yaml seeds: - name: waffle_types + description: Waffle types database: analytics schema: lookups columns: @@ -115,6 +117,7 @@ For non-standard CSV formats, configure parsing behavior with `csv_settings`: ```yaml seeds: - name: european_prices + description: European prices csv_settings: delimiter: ";" encoding: utf-8 diff --git a/website/src/content/docs/docs/concepts/snapshots.mdx b/website/src/content/docs/docs/concepts/snapshots.mdx index 119e0cf7c8..2e1ec9dd75 100644 --- a/website/src/content/docs/docs/concepts/snapshots.mdx +++ b/website/src/content/docs/docs/concepts/snapshots.mdx @@ -24,6 +24,7 @@ Use when your source has a reliable column recording when the entity changed. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -47,6 +48,7 @@ Use when the source does not have a reliable update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -75,6 +77,7 @@ Use for daily full exports or periodic snapshots without a business update times ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -98,6 +101,7 @@ Use for historical observations that include a business update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -123,6 +127,7 @@ Use for CDC tables, audit logs, or historical backfills where rows are individua ```sql MODEL ( + description "One row per order", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -228,6 +233,7 @@ The model `snapshot_full_refresh` field can only make the policy **stricter** th ```sql MODEL ( + description "Example model", materialized snapshot, ... snapshot_full_refresh deny, @@ -250,6 +256,7 @@ Snapshot models support the same audit system as other materializations. Audits ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -324,6 +331,7 @@ JOIN customer_snapshot c ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [user_id, role_id], snapshot_strategy check, @@ -341,6 +349,7 @@ FROM __source("user_role_daily") ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [product_id], snapshot_strategy timestamp, diff --git a/website/src/content/docs/docs/concepts/sources.mdx b/website/src/content/docs/docs/concepts/sources.mdx index 598f8e95ef..1c34ec0595 100644 --- a/website/src/content/docs/docs/concepts/sources.mdx +++ b/website/src/content/docs/docs/concepts/sources.mdx @@ -12,6 +12,7 @@ Point at an existing table or view in your warehouse: ```yaml sources: - name: raw_events + description: Events database: analytics schema: raw table: events @@ -26,6 +27,7 @@ Define source data inline as a SQL expression. No external tables or setup scrip ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT * FROM (VALUES (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), @@ -46,6 +48,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw_orders + description: Orders columns: - name: id audits: @@ -69,6 +72,7 @@ Type enforcement is implicit for sources. If any column declares a `type`, SQLBu ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name columns: @@ -91,6 +95,7 @@ Sources can be loaded by Python functions instead of pointing at existing tables ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -122,6 +127,7 @@ Configure freshness per source with a `freshness:` block: ```yaml sources: - name: raw_events + description: Events schema: raw table: events freshness: @@ -232,7 +238,7 @@ Use [`sqb freshness`](/docs/cli/freshness) to observe source freshness on demand | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | | `unique_key` | Merge key column(s) (required for `merge`) | | `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | -| `description` | Human-readable description | +| `description` | Required human-readable description ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | | `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | | `columns` | Column declarations with `name`, `type`, `nullable`, `description`, `meta`, and `audits` | diff --git a/website/src/content/docs/docs/integrations/dagster-reference.mdx b/website/src/content/docs/docs/integrations/dagster-reference.mdx index 7e88ef5a43..6f845dcf7a 100644 --- a/website/src/content/docs/docs/integrations/dagster-reference.mdx +++ b/website/src/content/docs/docs/integrations/dagster-reference.mdx @@ -207,6 +207,7 @@ A source can provide a stable Dagster-native key without replacing the translato ```yaml sources: - name: raw__orders + description: Orders meta: dagster: asset_key: [warehouse, raw, orders] diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 2f8e03e7fa..c1e4b8cc4e 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -45,6 +45,7 @@ dlt_sources: type: header_link resources: - name: raw_github_issues + description: Issues from the GitHub API write_disposition: append endpoint: path: "repos/${github_owner}/${github_repo}/issues" @@ -65,6 +66,7 @@ dlt_sources: credentials: "${postgres_connection_string}" resources: - name: raw_customers + description: Customers from the CRM database table: customers write_disposition: merge primary_key: id @@ -82,6 +84,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + description: Events from the event export files reader: csv write_disposition: append ``` @@ -93,6 +96,7 @@ dlt_sources: | Key | Description | |-----|-------------| | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | +| `description` | What the loaded rows are. Required ([Descriptions](/docs/concepts/models/configuration#descriptions)). | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | @@ -144,6 +148,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_github_issues(ctx: LoaderContext): + """Github issues.""" source = rest_api_source({ "client": { "base_url": "https://api.github.com/", @@ -178,6 +183,7 @@ Bind it to a source with `managed: true` in `sources/*.yml`: # sources/github.yml sources: - name: raw_github_issues + description: Github issues managed: true table: issues columns: diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index d218c3cb3e..47f0b7e9d0 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -38,6 +38,7 @@ Replicate a table from PostgreSQL into your local DuckDB project: # sources/raw.yml sources: - name: raw_orders + description: Orders table: orders ingestr: source_uri: "postgresql://user:pass@host:5432/mydb" @@ -53,6 +54,7 @@ Load Stripe charges with incremental merge on a primary key: ```yaml sources: - name: raw_stripe_charges + description: Stripe charges table: charges ingestr: source_uri: "stripe://${stripe_api_key}" @@ -95,6 +97,7 @@ All string fields in the `ingestr` block support SQLBuild project variable subst ```yaml sources: - name: raw_orders + description: Orders ingestr: source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" source_table: "public.orders" From aeeb41be0ed4bdf1e5f4d49476f7cf61b0303e25 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:46:08 +0100 Subject: [PATCH 4/4] ci: generate the base benchmark project with the base checkout's generator The same-runner compile ratio job generated one project with head's generator and compiled it with both builds, so any change that adds syntax to the generators (such as FUNCTION descriptions) made the base build reject the project. compare_compile_performance now accepts --base-root: the base project is written by the base checkout's own write_benchmark_project (run with the head interpreter and the base root on PYTHONPATH), head's project by head's generator, and each build compiles its own project. Base generation failures stop the comparison with the generator's stderr tail, and the report notes per-side generation. Without --base-root the single-project behaviour is kept. --- .github/workflows/verify.yml | 1 + .../_helpers/measure.py | 54 ++++++++++--- .../_helpers/report.py | 11 ++- .../compile_performance_ratio/constants.py | 8 ++ .../compile_performance_ratio/main/compare.py | 33 +++++++- .../compile_performance_ratio/_test_types.py | 10 +++ .../compile_performance_ratio/helpers.py | 61 +++++++++++++++ .../test_compare_compile_performance.py | 77 +++++++++++++++++++ 8 files changed, 239 insertions(+), 16 deletions(-) create mode 100644 tests/e2e/scripts/compile_performance_ratio/helpers.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9c4a376e34..fff57401b8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -460,6 +460,7 @@ jobs: --kind "${KIND}" \ --max-ratio "${MAX_RATIO}" \ --models 3000 \ + --base-root base \ --base-python base/.venv/bin/python \ --head-python .venv/bin/python diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 5e92b48b82..833667933e 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -1,4 +1,4 @@ -"""Generate a benchmark project once and time alternating base and head compiles.""" +"""Generate benchmark projects and time alternating base and head compiles.""" from __future__ import annotations @@ -13,9 +13,11 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio.constants import ( + BASE_GENERATOR_ENTRY, BASE_LABEL, COMPILE_ENTRY, DENSE_KIND, + ERROR_TAIL_CHARACTERS, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, FRESH_AUDIT_SHARE, @@ -25,6 +27,7 @@ FRESH_SOURCE_SHARE, FRESH_TEST_SHARE, HEAD_LABEL, + PYTHONPATH_KEY, REPORTED_PHASES, ) from scripts.compile_performance_ratio.exceptions import CompileComparisonError @@ -52,23 +55,47 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non ) +def write_base_benchmark_project( + *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int +) -> None: + """Write the base project with the base checkout's own generator, run by `python`.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)], + cwd=base_root, + capture_output=True, + text=True, + env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)}, + check=False, + ) + if completed.returncode != 0: + raise CompileComparisonError( + f"base project generation in {base_root} failed with exit {completed.returncode}: " + f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}" + ) + + def compare_builds( *, kind: str, models: int, - project_dir: Path, + base_project_dir: Path, + head_project_dir: Path, base_python: Path, head_python: Path, runs: int, ) -> CompileComparison: """Alternate base and head cold compiles after one untimed warm-up compile per build.""" - builds: tuple[tuple[str, Path], ...] = ((BASE_LABEL, base_python), (HEAD_LABEL, head_python)) - for label, python in builds: + builds: tuple[tuple[str, Path, Path], ...] = ( + (BASE_LABEL, base_python, base_project_dir), + (HEAD_LABEL, head_python, head_project_dir), + ) + for label, python, project_dir in builds: _compile_once(label=label, python=python, project_dir=project_dir) results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []} for _ in range(runs): - for label, python in builds: + for label, python, project_dir in builds: results[label].append( _compile_once(label=label, python=python, project_dir=project_dir) ) @@ -88,11 +115,7 @@ def compare_builds( def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: shutil.rmtree(project_dir / "target", ignore_errors=True) - environment: dict[str, str] = { - key: value - for key, value in os.environ.items() - if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) - } + environment: dict[str, str] = _compile_environment() before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) started: float = time.perf_counter() completed: subprocess.CompletedProcess[str] = subprocess.run( @@ -116,7 +139,8 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) if completed.returncode != 0: raise CompileComparisonError( - f"{label} compile failed with exit {completed.returncode}: {completed.stderr[-2000:]}" + f"{label} compile failed with exit {completed.returncode}: " + f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}" ) payload: dict[str, object] = json.loads(completed.stdout) timings: object = payload.get("compile_timings", {}) @@ -132,6 +156,14 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: ) +def _compile_environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } + + def _median_phases(*, runs: list[CompileRun]) -> dict[str, float]: medians: dict[str, float] = {} for phase in REPORTED_PHASES: diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 3df30ab101..7324d7d66a 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -9,14 +9,23 @@ from scripts.compile_performance_ratio.models import CompileComparison -def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: float) -> str: +def comparison_markdown( + *, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool +) -> str: """Summarize medians, ratios and per-phase timings as a Markdown table.""" + generation: str = ( + "Projects generated per side: base with the base checkout's generator, head with head's." + if per_side_projects + else "One project generated with head's generator and compiled by both builds." + ) lines: list[str] = [ f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", "", f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", "", + generation, + "", "| Metric | Base | Head | Ratio |", "|---|---:|---:|---:|", f"| Wall (s) | {comparison.base_wall_seconds:.2f} | " diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 3bd0a44012..3698f796a3 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -10,6 +10,14 @@ EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"}) EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +BASE_GENERATOR_ENTRY: str = ( + "import sys; from pathlib import Path; " + "from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; " + "write_benchmark_project(" + "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))" +) +PYTHONPATH_KEY: str = "PYTHONPATH" +ERROR_TAIL_CHARACTERS: int = 2000 FRESH_SOURCE_SHARE: float = 713 / 3000 FRESH_SEED_SHARE: float = 141 / 3000 FRESH_FUNCTION_SHARE: float = 71 / 3000 diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index c1542bbaa9..a38382e0f3 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -10,6 +10,7 @@ from scripts.compile_performance_ratio._helpers.measure import ( compare_builds, + write_base_benchmark_project, write_benchmark_project, ) from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown @@ -26,21 +27,39 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: args: argparse.Namespace = _parse_args(argv) summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") + per_side_projects: bool = args.base_root is not None with tempfile.TemporaryDirectory(prefix="sqlbuild-ratio-") as root: - project_dir: Path = Path(root) / f"{args.kind}_{args.models}" + project_name: str = f"{args.kind}_{args.models}" + head_project_dir: Path = Path(root) / "head" / project_name + base_project_dir: Path = ( + Path(root) / "base" / project_name if per_side_projects else head_project_dir + ) print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr) - write_benchmark_project(kind=args.kind, project_dir=project_dir, models=args.models) + write_benchmark_project(kind=args.kind, project_dir=head_project_dir, models=args.models) + if args.base_root is not None: + print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr) + write_base_benchmark_project( + base_root=args.base_root.absolute(), + python=args.head_python.absolute(), + kind=args.kind, + project_dir=base_project_dir, + models=args.models, + ) print(f"Comparing base and head over {args.runs} alternating runs", file=sys.stderr) comparison: CompileComparison = compare_builds( kind=args.kind, models=args.models, - project_dir=project_dir, + base_project_dir=base_project_dir, + head_project_dir=head_project_dir, base_python=args.base_python, head_python=args.head_python, runs=args.runs, ) markdown: str = comparison_markdown( - comparison=comparison, runs=args.runs, max_ratio=args.max_ratio + comparison=comparison, + runs=args.runs, + max_ratio=args.max_ratio, + per_side_projects=per_side_projects, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -64,6 +83,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--models", type=int, required=True) parser.add_argument("--base-python", type=Path, required=True) parser.add_argument("--head-python", type=Path, required=True) + parser.add_argument( + "--base-root", + type=Path, + default=None, + help="Base checkout whose own generator writes the project the base build compiles.", + ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) return parser.parse_args(argv) diff --git a/tests/e2e/scripts/compile_performance_ratio/_test_types.py b/tests/e2e/scripts/compile_performance_ratio/_test_types.py index a58f84cf90..8f517caae3 100644 --- a/tests/e2e/scripts/compile_performance_ratio/_test_types.py +++ b/tests/e2e/scripts/compile_performance_ratio/_test_types.py @@ -13,3 +13,13 @@ class CompilePerformanceRatioTestCase: max_ratio: str expected_return_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PerSideCompilePerformanceRatioTestCase: + description: str + base_generator: str + expected_return_code: int + expected_fragments: tuple[str, ...] + expected_base_projects: tuple[str, ...] + expected_head_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/compile_performance_ratio/helpers.py b/tests/e2e/scripts/compile_performance_ratio/helpers.py new file mode 100644 index 0000000000..7a3b7f1acf --- /dev/null +++ b/tests/e2e/scripts/compile_performance_ratio/helpers.py @@ -0,0 +1,61 @@ +"""Fixtures for per-side compile performance ratio comparisons.""" + +from __future__ import annotations + +import stat +import sys +from pathlib import Path + +MARKER_GENERATOR: str = """from pathlib import Path + + +def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None: + del models + (project_dir / "models").mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text('name = "orders"\\nadapter = "duckdb"\\n') + (project_dir / "models" / "base_marker.sql").write_text( + 'MODEL (description "Base marker");\\n\\nSELECT 1 AS id\\n' + ) + (project_dir / "base_marker.txt").write_text(kind) +""" +FAILING_GENERATOR: str = """def write_benchmark_project(*, kind, project_dir, models): + raise RuntimeError("base generator exploded") +""" +_LOGGING_PYTHON: str = """#!/usr/bin/env bash +project="" +previous="" +for argument in "$@"; do + if [ "$previous" = "--project-dir" ]; then project="$argument"; fi + previous="$argument" +done +if [ -n "$project" ]; then + if [ -f "$project/base_marker.txt" ]; then echo base >> "{log}"; else echo head >> "{log}"; fi +fi +exec "{python}" "$@" +""" + + +def write_fake_base_root(*, root: Path, generator: str) -> Path: + """Write a base checkout whose only content is a benchmark generator module.""" + + helpers: Path = root / "scripts" / "compile_performance_ratio" / "_helpers" + helpers.mkdir(parents=True) + for package in (root / "scripts", helpers.parent, helpers): + (package / "__init__.py").write_text("", encoding="utf-8") + (helpers / "measure.py").write_text(generator, encoding="utf-8") + return root + + +def write_logging_python(*, path: Path, log: Path) -> Path: + """Write an interpreter wrapper logging which project each compile received.""" + + path.write_text(_LOGGING_PYTHON.format(log=log, python=sys.executable), encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + log.touch() + return path + + +def logged_projects(log: Path) -> tuple[str, ...]: + """Return the project kind each logged compile received, in order.""" + + return tuple(log.read_text(encoding="utf-8").split()) diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py index e322a59273..407b597179 100644 --- a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -11,6 +11,14 @@ from tests.e2e.scripts.compile_performance_ratio._test_types import ( CompilePerformanceRatioTestCase, + PerSideCompilePerformanceRatioTestCase, +) +from tests.e2e.scripts.compile_performance_ratio.helpers import ( + FAILING_GENERATOR, + MARKER_GENERATOR, + logged_projects, + write_fake_base_root, + write_logging_python, ) _REPO_ROOT: Path = Path(__file__).resolve().parents[4] @@ -74,5 +82,74 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio assert fragment in output +@pytest.mark.parametrize( + "test_case", + ( + PerSideCompilePerformanceRatioTestCase( + description="each build compiles the project its own generator wrote", + base_generator=MARKER_GENERATOR, + expected_return_code=0, + expected_fragments=("Projects generated per side",), + expected_base_projects=("base", "base"), + expected_head_projects=("head", "head"), + ), + PerSideCompilePerformanceRatioTestCase( + description="a failing base generator stops before any compile", + base_generator=FAILING_GENERATOR, + expected_return_code=1, + expected_fragments=("base project generation in", "base generator exploded"), + expected_base_projects=(), + expected_head_projects=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_base_root_when_comparing_compile_performance_then_each_side_uses_its_project( + test_case: PerSideCompilePerformanceRatioTestCase, tmp_path: Path +) -> None: + base_root: Path = write_fake_base_root( + root=tmp_path / "base", generator=test_case.base_generator + ) + base_log: Path = tmp_path / "base.log" + head_log: Path = tmp_path / "head.log" + base_python: Path = write_logging_python(path=tmp_path / "base-python", log=base_log) + head_python: Path = write_logging_python(path=tmp_path / "head-python", log=head_log) + environment: dict[str, str] = dict(os.environ) + _ = environment.pop("GITHUB_STEP_SUMMARY", None) + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-m", + "scripts.compare_compile_performance", + "--kind", + "dense", + "--models", + "20", + "--runs", + "1", + "--base-root", + str(base_root), + "--base-python", + str(base_python), + "--head-python", + str(head_python), + "--max-ratio", + "1000", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + env=environment, + check=False, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert logged_projects(base_log) == test_case.expected_base_projects + assert logged_projects(head_log) == test_case.expected_head_projects + + if __name__ == "__main__": pytest.main([__file__, "-vv"])