diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 9c4a376e34..fff57401b8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -460,6 +460,7 @@ jobs: --kind "${KIND}" \ --max-ratio "${MAX_RATIO}" \ --models 3000 \ + --base-root base \ --base-python base/.venv/bin/python \ --head-python .venv/bin/python diff --git a/README.md b/README.md index ff7ef67836..3dc9d8eb68 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ A model is a SQL file with a `MODEL()` header and a `SELECT`: ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (audits [not_null, unique]), diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs index 1545d99e54..a503f7d2e2 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs @@ -191,32 +191,6 @@ fn given_sql_test_facts_when_evaluating_project_rules_then_returns_expected_faul "tests/unit/test_orders__paid.sql", ], }, - test_types::SqlTestRulesTestCase { - description: "scenario descriptions reject generic numbering", - code: "SQBRTEST105", - tests: json!([]), - scenarios: json!([ - { - "source_path": "tests/scenarios/orders__paid.sql", - "ownership_root": "tests/scenarios", "name": "orders__paid", - "description": "Paid orders remain visible", "expected_model_names": [], - "assertion_names": [], "assertion_target_model_names": [], - "target_model_names": [] - }, - { - "source_path": "tests/scenarios/orders__case_1.sql", - "ownership_root": "tests/scenarios", "name": "orders__case_1", - "description": "case 1", "expected_model_names": [], - "assertion_names": [], "assertion_target_model_names": [], - "target_model_names": [] - } - ]), - scope_index: helpers::scope_index(), - config: json!({}), - expected_fault_count: 1, - expected_evaluated_models: 0, - expected_paths: &["tests/scenarios/orders__case_1.sql"], - }, test_types::SqlTestRulesTestCase { description: "path-scoped ignore suppresses a test rule finding", code: "SQBRTEST104", diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs index f5105ad6cd..5d5741fb04 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs @@ -38,7 +38,6 @@ macro_rules! rule { | "SQBRTEST102" | "SQBRTEST103" | "SQBRTEST104" - | "SQBRTEST105" | "SQBRTEST203" | "SQBRTEST301" ), @@ -294,13 +293,6 @@ pub(crate) fn catalogue() -> Vec { "every SQL unit-test block must have a target-aware subject__expected_behavior name", "Add name \"__\" to this TEST header.", ), - rule!( - "SQBRTEST105", - "tests", - "scenario-business-description", - "scenario descriptions must identify a concrete business behavior", - "Write a non-generic SCENARIO description that states the business behavior under test.", - ), rule!( "SQBRTEST201", "tests", diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs index a68c859612..6743a60084 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs @@ -90,9 +90,6 @@ pub(crate) fn evaluate_project( if let Some(rule) = evaluation.selected.get("SQBRTEST104") { faults.extend(structured_names(rule, &evaluation, &tests)); } - if let Some(rule) = evaluation.selected.get("SQBRTEST105") { - faults.extend(scenario_descriptions(rule, &scenarios)); - } if let Some(rule) = evaluation.selected.get(EMPTY_INPUT_RULE_CODE) { faults.extend(empty_input_only_faults( rule, @@ -270,27 +267,6 @@ fn structured_names( faults } -fn scenario_descriptions(rule: &RuleMetadata, scenarios: &[&SqlScenarioFact]) -> Vec { - let mut faults: Vec = Vec::new(); - for scenario in scenarios { - let description = scenario.description.as_deref().unwrap_or("").trim(); - let behavior = file_stem(&scenario.source_path) - .split_once("__") - .map(|(_, value)| value) - .unwrap_or(""); - if description.is_empty() || generic(description) || generic(behavior) { - faults.push(path_fault( - rule, - &scenario.source_path, - "scenario description or filename uses a generic case label".into(), - "Write a concrete business description and a __.sql filename." - .into(), - )); - } - } - faults -} - fn expected_parent( evaluation: &ProjectEvaluationRequest<'_>, test: &SqlTestFact, diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index a0ca920076..0d7ef833c5 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -156,6 +156,7 @@ def _write_sources(*, project_dir: Path, source_count: int) -> None: entries.extend( ( f" - name: orders_{index:05d}", + f" description: Generated source orders_{index:05d}.", " contract: enforced", f' expression: "(SELECT 1 AS id, CAST(1 AS DOUBLE) AS amount{extra})"', " columns:", @@ -248,9 +249,8 @@ def _model_sql(*, index: int, source_count: int, function_count: int, macro_coun f'description "Order quantity measure"{audit})' ) header: str = ( - "MODEL (materialized table, contract enforced, columns (\n" - + ",\n".join(declarations) - + "\n));\n" + 'MODEL (description "Generated dense model.", materialized table, contract enforced, ' + "columns (\n" + ",\n".join(declarations) + "\n));\n" ) source_index: int = _source_index(index=index, source_count=source_count) relation: str = ( diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_project.py b/scripts/cold_compile_performance/_helpers/diagnostic_project.py index 93d4ffbe2b..71a8d54044 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_project.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_project.py @@ -16,7 +16,8 @@ def write_diagnostic_project( columns: list[str] = [f"amount_{index:03}" for index in range(width)] declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) (models / "orders.sql").write_text( - f"MODEL (materialized view, contract enforced, columns ({declarations}));\nSELECT " + "MODEL (description 'Test model orders.', " + f"materialized view, contract enforced, columns ({declarations}));\nSELECT " + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) ) for model_index in range(model_count): @@ -31,12 +32,14 @@ def write_diagnostic_project( conversions: list[str] = [ f"{columns[index % width]} = 'shipped' AS risky_{index:03}" for index in range(50) ] - sql: str = "MODEL (materialized view);\nWITH\n" + ",\n".join(ctes) + sql: str = "MODEL (description 'Test model.', materialized view);\nWITH\n" + ",\n".join( + ctes + ) sql += "\nSELECT\n" + ",\n".join((*errors, *conversions)) + f"\nFROM {previous}\n" name = f"large_orders_{model_index}" (models / f"{name}.sql").write_text(sql) (models / f"downstream_orders_{model_index}.sql").write_text( - "MODEL (materialized view);\nSELECT " + "MODEL (description 'Test model.', materialized view);\nSELECT " + ", ".join(f"broken_{index:03} + 1 AS result_{index:03}" for index in range(50)) + f' FROM __ref("{name}")' ) @@ -52,9 +55,13 @@ def write_cascade_project(*, project_dir: Path, depth: int, width: int = 50) -> ) columns: list[str] = [f"amount_{index:03}" for index in range(width)] declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) - header: str = f"MODEL (materialized view, contract enforced, columns ({declarations}));\n" + header: str = ( + "MODEL (description 'Test model.', " + f"materialized view, contract enforced, columns ({declarations}));\n" + ) (models / "input_orders.sql").write_text( - f"MODEL (materialized view, contract enforced, columns ({declarations}, " + "MODEL (description 'Test model input_orders.', " + f"materialized view, contract enforced, columns ({declarations}, " "ordered_at (type TIMESTAMP)));\nSELECT " + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) + ", CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" @@ -94,7 +101,7 @@ def write_function_name_project(*, project_dir: Path, model_count: int, calls: i ) ) (models / f"wide_products_{model_index}.sql").write_text( - "MODEL (database warehouse, schema analytics);\nSELECT\n" + "MODEL (description 'Test model.', database warehouse, schema analytics);\nSELECT\n" + ",\n".join(projections) + "\nFROM (SELECT 'anvil' AS sku) AS products\nWHERE NOT (sku IN ('a', 'b'))\n" ) @@ -109,7 +116,8 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int 'name = "dense_diagnostic_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' ) (models / "orders.sql").write_text( - "MODEL (materialized view, contract enforced, columns (order_id (type INTEGER), " + "MODEL (description 'Test model orders.', " + "materialized view, contract enforced, columns (order_id (type INTEGER), " "status (type VARCHAR)));\nSELECT CAST(1 AS INTEGER) AS order_id, " "CAST('open' AS VARCHAR) AS status" ) @@ -120,7 +128,7 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int for index in range(diagnostics) ] (models / "order_diagnostics.sql").write_text( - "MODEL (materialized view);\nSELECT\n" + "MODEL (description 'Test model order_diagnostics.', materialized view);\nSELECT\n" + ",\n".join(projections) + '\nFROM __ref("orders") AS o\n' ) diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 5e92b48b82..833667933e 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -1,4 +1,4 @@ -"""Generate a benchmark project once and time alternating base and head compiles.""" +"""Generate benchmark projects and time alternating base and head compiles.""" from __future__ import annotations @@ -13,9 +13,11 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio.constants import ( + BASE_GENERATOR_ENTRY, BASE_LABEL, COMPILE_ENTRY, DENSE_KIND, + ERROR_TAIL_CHARACTERS, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, FRESH_AUDIT_SHARE, @@ -25,6 +27,7 @@ FRESH_SOURCE_SHARE, FRESH_TEST_SHARE, HEAD_LABEL, + PYTHONPATH_KEY, REPORTED_PHASES, ) from scripts.compile_performance_ratio.exceptions import CompileComparisonError @@ -52,23 +55,47 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non ) +def write_base_benchmark_project( + *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int +) -> None: + """Write the base project with the base checkout's own generator, run by `python`.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)], + cwd=base_root, + capture_output=True, + text=True, + env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)}, + check=False, + ) + if completed.returncode != 0: + raise CompileComparisonError( + f"base project generation in {base_root} failed with exit {completed.returncode}: " + f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}" + ) + + def compare_builds( *, kind: str, models: int, - project_dir: Path, + base_project_dir: Path, + head_project_dir: Path, base_python: Path, head_python: Path, runs: int, ) -> CompileComparison: """Alternate base and head cold compiles after one untimed warm-up compile per build.""" - builds: tuple[tuple[str, Path], ...] = ((BASE_LABEL, base_python), (HEAD_LABEL, head_python)) - for label, python in builds: + builds: tuple[tuple[str, Path, Path], ...] = ( + (BASE_LABEL, base_python, base_project_dir), + (HEAD_LABEL, head_python, head_project_dir), + ) + for label, python, project_dir in builds: _compile_once(label=label, python=python, project_dir=project_dir) results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []} for _ in range(runs): - for label, python in builds: + for label, python, project_dir in builds: results[label].append( _compile_once(label=label, python=python, project_dir=project_dir) ) @@ -88,11 +115,7 @@ def compare_builds( def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: shutil.rmtree(project_dir / "target", ignore_errors=True) - environment: dict[str, str] = { - key: value - for key, value in os.environ.items() - if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) - } + environment: dict[str, str] = _compile_environment() before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) started: float = time.perf_counter() completed: subprocess.CompletedProcess[str] = subprocess.run( @@ -116,7 +139,8 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) if completed.returncode != 0: raise CompileComparisonError( - f"{label} compile failed with exit {completed.returncode}: {completed.stderr[-2000:]}" + f"{label} compile failed with exit {completed.returncode}: " + f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}" ) payload: dict[str, object] = json.loads(completed.stdout) timings: object = payload.get("compile_timings", {}) @@ -132,6 +156,14 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: ) +def _compile_environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } + + def _median_phases(*, runs: list[CompileRun]) -> dict[str, float]: medians: dict[str, float] = {} for phase in REPORTED_PHASES: diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 3df30ab101..7324d7d66a 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -9,14 +9,23 @@ from scripts.compile_performance_ratio.models import CompileComparison -def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: float) -> str: +def comparison_markdown( + *, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool +) -> str: """Summarize medians, ratios and per-phase timings as a Markdown table.""" + generation: str = ( + "Projects generated per side: base with the base checkout's generator, head with head's." + if per_side_projects + else "One project generated with head's generator and compiled by both builds." + ) lines: list[str] = [ f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", "", f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", "", + generation, + "", "| Metric | Base | Head | Ratio |", "|---|---:|---:|---:|", f"| Wall (s) | {comparison.base_wall_seconds:.2f} | " diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 3bd0a44012..3698f796a3 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -10,6 +10,14 @@ EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"}) EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +BASE_GENERATOR_ENTRY: str = ( + "import sys; from pathlib import Path; " + "from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; " + "write_benchmark_project(" + "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))" +) +PYTHONPATH_KEY: str = "PYTHONPATH" +ERROR_TAIL_CHARACTERS: int = 2000 FRESH_SOURCE_SHARE: float = 713 / 3000 FRESH_SEED_SHARE: float = 141 / 3000 FRESH_FUNCTION_SHARE: float = 71 / 3000 diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index c1542bbaa9..a38382e0f3 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -10,6 +10,7 @@ from scripts.compile_performance_ratio._helpers.measure import ( compare_builds, + write_base_benchmark_project, write_benchmark_project, ) from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown @@ -26,21 +27,39 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: args: argparse.Namespace = _parse_args(argv) summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") + per_side_projects: bool = args.base_root is not None with tempfile.TemporaryDirectory(prefix="sqlbuild-ratio-") as root: - project_dir: Path = Path(root) / f"{args.kind}_{args.models}" + project_name: str = f"{args.kind}_{args.models}" + head_project_dir: Path = Path(root) / "head" / project_name + base_project_dir: Path = ( + Path(root) / "base" / project_name if per_side_projects else head_project_dir + ) print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr) - write_benchmark_project(kind=args.kind, project_dir=project_dir, models=args.models) + write_benchmark_project(kind=args.kind, project_dir=head_project_dir, models=args.models) + if args.base_root is not None: + print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr) + write_base_benchmark_project( + base_root=args.base_root.absolute(), + python=args.head_python.absolute(), + kind=args.kind, + project_dir=base_project_dir, + models=args.models, + ) print(f"Comparing base and head over {args.runs} alternating runs", file=sys.stderr) comparison: CompileComparison = compare_builds( kind=args.kind, models=args.models, - project_dir=project_dir, + base_project_dir=base_project_dir, + head_project_dir=head_project_dir, base_python=args.base_python, head_python=args.head_python, runs=args.runs, ) markdown: str = comparison_markdown( - comparison=comparison, runs=args.runs, max_ratio=args.max_ratio + comparison=comparison, + runs=args.runs, + max_ratio=args.max_ratio, + per_side_projects=per_side_projects, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -64,6 +83,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--models", type=int, required=True) parser.add_argument("--base-python", type=Path, required=True) parser.add_argument("--head-python", type=Path, required=True) + parser.add_argument( + "--base-root", + type=Path, + default=None, + help="Base checkout whose own generator writes the project the base build compiles.", + ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) return parser.parse_args(argv) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 5894fca18c..fa13f57633 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -17,6 +17,9 @@ plan, and ships tools for proving a change is correct. Use those tools; do not s - Macros are plain Python functions called as `@macro_name("arg")`. Constants and enums are `@const("name")` and `@enum("name").MEMBER`. - Config, column schemas, contracts and audits live in the `MODEL()` header, not YAML. +- Every model, scenario, seed, source, function, named hook, Python node and provider needs a + non-empty description (`MODEL (description "...")`, `description:` in YAML, a docstring or + `description=` in Python). A missing one is a `P010` compile error that shows the exact line to add. - `sqb compile` is offline: syntax, per-input binding, grouping semantics, contracts, metadata, column lineage and Rules run before warehouse execution. Explicit projections close output names even over open inputs. DuckDB/MotherDuck, PostgreSQL, Snowflake, and BigQuery enable expression @@ -36,6 +39,7 @@ plan, and ships tools for proving a change is correct. Use those tools; do not s ```sql -- models/marts/fact_orders.sql MODEL ( + description "One row per order with its payment total", materialized table, columns ( order_id (audits [not_null, unique]), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 8af247c1a8..56243ac9ac 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -47,6 +47,14 @@ Configure the maximum physical line width in `sqlbuild_project.toml` (the defaul line_width = 100 ``` +`MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; +`sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it +before compile does (see [Descriptions](../concepts/models/configuration.md#descriptions)). A model +described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +Format does not render templates, so a templated description that renders empty passes format and +is reported by `sqb compile`. +Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. + Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks are normalized as spaces, while blank lines preserve paragraph boundaries. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md index 992fffc89f..762d5b6339 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md @@ -150,7 +150,7 @@ def formatted_order_total(expression: str) -> str: ```sql -- models/commerce/orders.sql -MODEL(); +MODEL(description "Orders"); SELECT @formatted_order_total("subtotal") AS total ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index f48620c5a6..833b3dbd9e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -67,6 +67,7 @@ Attach threshold and sample policy where the audit is used: ```sql MODEL ( + description "Valid order rate", audits [ valid_order_rate ( condition "order_id IS NOT NULL", @@ -130,6 +131,7 @@ Attach them in the `MODEL()` header like any generic audit: ```sql MODEL ( + description "One row per order", materialized view, tags [staging], columns ( @@ -212,6 +214,7 @@ that is not visible from there is a compile error. See ```sql MODEL ( + description "One row per order", materialized table, audits [ expression_is_true ( @@ -320,6 +323,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw__orders + description: Orders columns: - name: id audits: @@ -344,6 +348,7 @@ Seeds use the same YAML audit syntax as sources, at table and column level: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -397,6 +402,7 @@ Audits on incremental models can run at different lifecycle phases: ```sql MODEL ( + description "One row per order", materialized incremental, ... columns ( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md index e2e1624208..9ab89d1670 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/dbt-compatibility/adding-sqlbuild-models.md @@ -31,6 +31,7 @@ SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")` ```sql MODEL ( + description "Finance orders built on the dbt fact table", tags [finance], columns (order_id (audits [not_null])), ); @@ -43,7 +44,7 @@ This resolves to the qualified warehouse table name from the dbt manifest (e.g. SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: ```sql -MODEL (tags [marts]); +MODEL (description "Mart view of downstream orders", tags [marts]); SELECT order_id FROM __ref("downstream_orders") ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md index 094d42f7ee..dd7aec6a84 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md @@ -93,6 +93,7 @@ inherited sample and request exhaustive comparison with `row_diff_sample_rows 0` ```sql MODEL ( + description "One row per order", materialized table, unique_key [order_id, line_id], row_diff_sample_rows 0, @@ -157,6 +158,7 @@ Numeric columns can have tolerance rules to avoid false positives from floating- ```sql MODEL ( + description "Example model", materialized incremental, ... row_diff_tolerances ( @@ -181,6 +183,7 @@ Columns that are expected to differ between the two sides (like timestamps or co ```sql MODEL ( + description "One row per order", materialized incremental, ... row_diff_exclude_columns [latest_order_status], diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md index 066c988fdf..46cd77daa9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/enums/model-contracts.md @@ -20,6 +20,7 @@ Use the enum name as the column type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md index 4043248d36..ef845f99e5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -24,11 +24,12 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `description`, `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line. `description` is required: a function without one fails compilation with `P010` (see [Descriptions](models/configuration.md#descriptions)): ```sql -- functions/sql/udf__is_completed_order.sql FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); @@ -50,9 +51,12 @@ from sqlbuild.functions import udf runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Whether an order status counts as completed.""" return order_status == "completed" ``` +The function docstring is its description; `@udf(description="...")` overrides it. One of them is required, or compilation fails with `P010`. The docstring is deployed with the function body, so editing it redeploys the function; a `description=` value does not. + Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. ### Using scalar UDFs @@ -75,6 +79,7 @@ Table functions return multiple rows and columns. They are written in SQL under ```sql -- functions/sql/table_fn__customer_orders.sql FUNCTION ( + description "Customer orders", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -165,6 +170,7 @@ The `@udf` decorator accepts these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| +| `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | | `returns` | Yes | SQL return type string | | `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 3bc4b8ee01..edc9f7eaab 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -28,6 +28,7 @@ Inserts new rows without modifying existing data. Optionally uses a cursor (read ```sql MODEL ( + description "One row per customer", materialized incremental, incremental_strategy append, cursor created_at, @@ -50,6 +51,7 @@ Deletes rows in the cursor range, then inserts the new delta. Requires either `c ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key [order_id], @@ -69,6 +71,7 @@ Upserts rows using a unique key. Matched rows are updated; unmatched rows are in ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -105,6 +108,7 @@ Use `full_refresh false` for models that must retain their normal cursor or micr ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, full_refresh false, @@ -144,6 +148,7 @@ When a model references multiple upstream inputs, `cursor_inputs` is required to ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -165,6 +170,7 @@ Cursor-based incremental models can read their effective interval with zero-argu ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -202,6 +208,7 @@ Lookback extends the start of the replay window backwards to re-process recent d ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, cursor event_date, @@ -227,6 +234,7 @@ Each batch has its own audit cycle: create delta, run delta audits, apply DML, a ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -265,6 +273,7 @@ microbatch_concurrency = true ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -284,6 +293,7 @@ Watermark microbatch models can declare what to do when their resolved range con ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -339,6 +349,7 @@ Alignment does two things: it floors the replay window edges to the coarsest gra ```sql -- Upstream: daily grain, 2d batches MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -353,6 +364,7 @@ MODEL ( -- Downstream: hourly grain, but aligns to day automatically MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -380,6 +392,7 @@ The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (s ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 978b7a6d77..7cf958daae 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -49,6 +49,7 @@ Use `@macro_name(...)` in a model query: ```sql MODEL ( + description "One row per order", materialized table, tags [marts], ); @@ -196,6 +197,7 @@ def grant_target(target: str) -> str: ```sql MODEL ( + description "Example model", post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], ); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md index b654019d97..ce3a77d0c4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/model-private-values.md @@ -11,6 +11,7 @@ project-wide namespace. ```sql MODEL ( + description "One row per order", enums ( _state [OPEN, CLOSED], ), @@ -51,6 +52,7 @@ Use `constant(...)` when a private constant needs an exact type or rendering cho ```sql MODEL ( + description "Example model", constants ( _usd_rate constant( type decimal, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md index eef9085a54..eff7b76918 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/column-migrations.md @@ -27,6 +27,7 @@ Add `migrate_from` to the renamed column in the model header, naming the old col ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key order_id, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 3dca51e3a8..bde6b83c91 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -9,6 +9,7 @@ Online: https://sqlbuild.com/docs/concepts/models/configuration/ ## Contents - Common fields +- Descriptions - SQL analysis - Table fields - Incremental fields @@ -35,7 +36,7 @@ which accepts any keys. |-------|-------------| | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | -| `description` | Human-readable model description | +| `description` | Required human-readable model description; see [Descriptions](#descriptions) | | `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](column-migrations.md)). On a table or view model that declares its own `migrate_from`, a column `migrate_from` only names the old column in the [compatibility view](migrations.md#old-names) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | @@ -52,6 +53,43 @@ which accepts any keys. | `dynamic_columns` | Named pivot column families with `pivot_column`, `value_column`, `aggregate`, `type`, and `name_pattern` | | `audit_factories` | Audit factories that generate audit cases for this model; see [Audits](../audits.md) | +## Descriptions + +Every named resource must have a non-empty description. A missing description is a `P010` compile +error, so `sqb plan` and `sqb build` stop too. There is no setting to turn the check off. One +compile reports every missing description, each at the resource's declaration with the exact text +to add: + +```text +error[P010]: model 'orders_summary' has no description + --> models/marts/orders_summary.sql:1:1 + = help: to describe model 'orders_summary', add this to the MODEL header in models/marts/orders_summary.sql: + MODEL ( + description "What one row of orders_summary represents", + ... + ); +``` + +| Resource | Where the description goes | +|----------|----------------------------| +| Model | `MODEL (description "...")`, or the `description` of its `model_schema` | +| Scenario | `SCENARIO (description "...")` | +| Seed | `description:` on its entry in `seeds/*.yml` | +| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| SQL function or table function | `FUNCTION (description "...")` | +| Python UDF | `@udf(description="...")` or the function docstring | +| Named SQL hook | `HOOK (description "...")` | +| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | +| Provider | The provider class docstring | + +SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a +description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a +model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it +does). `sqb format --check` also fails when a `MODEL`, `SCENARIO`, `FUNCTION` or `HOOK` +header has no description (a `MODEL` header that uses `model_schema` is left to compile), and +`sqb format` still moves a leading comment into an empty model description. + ## SQL analysis SQL analysis has project, invocation, and model gates: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md index 755de65810..567a86e07a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/contracts.md @@ -34,6 +34,7 @@ Under explicit mode, a model must declare `contract enforced` to activate shape ```sql MODEL ( + description "One row per order", materialized table, contract enforced, columns ( @@ -108,6 +109,7 @@ A column may use a declared enum as a portable logical domain type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md index 7fbfcd7bff..4f785a52d5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks.md @@ -89,6 +89,7 @@ Models may mix SQL and Python entries in one ordered hook list: ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md index 4aa0ad63fa..9441072c37 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -34,13 +34,14 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, channel="#data-builds"): + """Notify complete.""" ctx.log( f"Notify {channel}: " f"{ctx.model_name} completed during {ctx.phase}" ) ``` -By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks. +By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A hook needs a description: pass `description=` or give the function a docstring ([Descriptions](../configuration.md#descriptions)). A Python file may define multiple decorated hooks. ## Declared reads @@ -53,6 +54,7 @@ from sqlbuild.refs import model @hook(reads=model("dim_customers")) def refresh_lookup(ctx): + """Refresh lookup.""" customers = ctx.relation(model("dim_customers")) ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}") ``` @@ -90,6 +92,7 @@ Reference the hook by name and optionally pass keyword arguments: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ python( @@ -143,6 +146,7 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send( f"Model {ctx.model_name} built successfully" ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md index 753e1010d3..faf33a66bb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/sql.md @@ -38,7 +38,7 @@ GRANT SELECT ON @relation TO @role The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. -`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. +`HOOK()` accepts only a required, non-empty `description` ([Descriptions](../configuration.md#descriptions)). It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. @@ -50,6 +50,7 @@ Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` ```sql MODEL ( + description "Orders", materialized table, post_hooks [ sql( @@ -87,6 +88,7 @@ VALUES (@'relation', @'role') ```sql MODEL ( + description "Record access", post_hooks [ sql( "record_access", @@ -109,6 +111,7 @@ Use `inline_sql("...")` for SQL that is specific to one model: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), @@ -156,7 +159,7 @@ Context components can be combined into qualified identifiers without whitespace wrapper: ```sql -HOOK (); +HOOK (description "Create the reconstruct_book helper function"); CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() RETURNS INTEGER @@ -171,6 +174,7 @@ SQL hooks reference other resources the same way model SQL does, with `__ref(".. ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'), @@ -184,7 +188,7 @@ SELECT order_id FROM __ref("stg_orders") **`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`** ```sql -HOOK (); +HOOK (description "Record counts"); @count_rows(__ref("stg_orders")) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index b4470341b9..05ff176ab4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -13,7 +13,7 @@ The `materialized` field selects how a model becomes a warehouse relation. Creates or replaces a database view on each build: ```sql -MODEL (materialized view); +MODEL (description "One row per order", materialized view); SELECT id AS order_id, customer_id, status FROM __source("raw__orders") @@ -26,7 +26,7 @@ View audits run after the view has been replaced. A failing audit marks the buil By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. ```sql -MODEL (materialized table); +MODEL (description "One row per order", materialized table); SELECT customer_id, COUNT(*) AS total_orders FROM __ref("stg_orders") @@ -41,6 +41,7 @@ On a normal incremental run, applies append, delete/insert, or merge DML from a ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -60,6 +61,7 @@ Maintains historical row versions with SCD Type 2 semantics: ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -78,6 +80,7 @@ A project-local Python materialization can manage specialized persistence with a ```sql MODEL ( + description "One row per order", materialized partition_tracked, placeholders ( partition_start "'2026-04-01'", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index b65efc1bd3..3576e4d1c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -29,6 +29,7 @@ To declare a migration by hand, add `migrate_from` to the renamed model's header ```sql MODEL ( + description "One row per order", materialized incremental, migrate_from stg_orders, cursor order_date, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md index 348bbcfd67..02ac567e6a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/type-enforcement.md @@ -14,6 +14,7 @@ For a SQL model, declaring a type on any inline or reusable-schema column enable ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (type INTEGER), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md index 8140ad5614..34a2cf3d07 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -28,6 +28,7 @@ from sqlbuild.assets import asset, AssetContext @asset def orders_export(ctx: AssetContext): + """Orders export.""" path = write_orders_csv() return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) ``` @@ -41,6 +42,7 @@ Assets record whether they actually produced an artifact via `materialized`: ```python @asset def orders_export(ctx: AssetContext): + """Orders export.""" if nothing_changed(): return ctx.result(metadata={"status": "unchanged"}, materialized=False) return ctx.result(metadata={"path": export()}, materialized=True) @@ -61,6 +63,7 @@ from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_dashboard(ctx): + """Orders dashboard.""" result = ctx.result_of(export_orders) return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) ``` @@ -73,6 +76,7 @@ from sqlbuild.refs import model @asset(depends_on=model("fact_orders")) def orders_extract(ctx): + """Orders extract.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] return ctx.result(metadata={"rows": rows}, materialized=True) @@ -94,6 +98,7 @@ Assets can declare a schema for catalog and lineage purposes. This does not enfo }, ) def orders_export(ctx): + """Orders export.""" ... ``` @@ -112,6 +117,7 @@ from sqlbuild.retries import RetryPolicy @asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) def export(ctx): + """Export the order summary.""" if not ready(): return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT return ctx.result(metadata={"path": do_export()}, materialized=True) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 85a07e2094..eca32a9470 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -31,6 +31,7 @@ from python.tasks.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): + """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: return ctx.fail("no orders exported") @@ -47,6 +48,7 @@ from sqlbuild.refs import model @check(depends_on=model("fact_orders")) def orders_present(ctx): + """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] return ctx.pass_() if count else ctx.fail("no orders") @@ -61,6 +63,7 @@ Return a result through the context helpers, or a bool shorthand: ```python @check(depends_on=orders_asset) def rows_present(ctx): + """Rows present.""" return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail ``` @@ -81,6 +84,7 @@ Returning `None` is not allowed - checks must be explicit. ```python @check(depends_on=export_orders, severity="warn") def orders_freshness(ctx): + """Orders freshness.""" if stale(): return ctx.fail("export is stale") # recorded as a warning, does not fail the build return ctx.pass_() diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index f5c8a2e866..e0cdfed182 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -27,6 +27,7 @@ def export_assets(): for table in TABLES: @asset(name=f"export_{table}", tags=("export",)) def export(ctx, table=table): + """Export one table to object storage.""" return ctx.result(metadata={"table": table}, materialized=True) nodes.append(export) return nodes @@ -57,14 +58,17 @@ from sqlbuild.checks import check def orders_pipeline(): @loader(name="raw_orders") def load(ctx): + """Load orders from the storefront API.""" return fetch_orders() @asset(name="orders_export", depends_on=load) def export(ctx): + """Export the order summary.""" return ctx.result(materialized=True) @check(depends_on=export) def orders_export_check(ctx): + """Check that the orders export is ready.""" return ctx.pass_("export ready") return [load, export, orders_export_check] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index d3a6d8ac74..0cd4e2c50f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -43,6 +43,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customers from the CRM export.""" return [ {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, @@ -58,6 +59,7 @@ Declare a managed source in `sources/*.yml`. A managed source is bound to the lo ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -88,6 +90,7 @@ Full replace. The target is dropped and recreated from the loader output on ever ```yaml sources: - name: raw_countries + description: Countries managed: true write_strategy: table columns: @@ -104,6 +107,7 @@ Insert all returned rows into the target. No deduplication. ```yaml sources: - name: raw_webhook_events + description: Webhook events managed: true write_strategy: append columns: @@ -120,6 +124,7 @@ Delete rows in the cursor range, then insert replacements. Requires `cursor_colu ```yaml sources: - name: raw_order_events + description: Order events managed: true write_strategy: delete_insert cursor_column: event_at @@ -137,6 +142,7 @@ The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_colu ```python @loader def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" if ctx.current_cursor_value is None: return fetch_all_events() return fetch_events_since(ctx.current_cursor_value) @@ -149,6 +155,7 @@ Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: merge unique_key: customer_id @@ -171,6 +178,7 @@ If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader ```python @loader def raw_status(ctx: LoaderContext) -> None: + """Load the order status lookup.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " @@ -183,6 +191,7 @@ The source is still declared as managed, just without a `write_strategy`: ```yaml sources: - name: raw_status + description: Order status lookup managed: true columns: - name: status_id @@ -253,6 +262,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" return [ {"account_id": 1, "account_name": "Pawnee Parks"}, {"account_id": 2, "account_name": "Eagleton"}, @@ -260,6 +270,7 @@ def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: @loader(depends_on=[raw_accounts]) def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" accounts = ctx.loader(raw_accounts) rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") return [ @@ -277,6 +288,7 @@ Intermediate loaders (those referenced only via `depends_on`, with no managed so ```python @loader(destination="staging.shared_accounts") def raw_accounts(ctx: LoaderContext): + """Load customer accounts.""" ... ``` @@ -293,6 +305,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | `unique_key` | Column(s) used as the merge key (string or list of strings) | | `columns` | Column specifications with name, type, nullable, and description | | `contract` | `enforced` or `none` | +| `description` | What the loaded rows are; defaults to the function docstring. Required unless the source YAML entry has a `description` ([Descriptions](../models/configuration.md#descriptions)) | ## Auto-load during builds diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 95e5a8cef0..38cb8e3262 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -94,7 +94,7 @@ Every decorator accepts the same organizational metadata: | `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | -| `description` | Human-readable docs (defaults to the function docstring) | +| `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](../models/configuration.md#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | `@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. @@ -135,6 +135,7 @@ Tasks and assets return through `ctx.result(...)`: ```python @task def export_orders(ctx): + """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 1ae87b4d78..335a237f59 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -30,6 +30,7 @@ Create a Python file under `providers/` in your project. A provider is a class t from sqlbuild.providers import Provider class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str endpoint: str = "https://api.example.com" @@ -40,6 +41,8 @@ class WarehouseClient(Provider): self.session.close() ``` +The class docstring is the provider's description and is required; a provider without one fails compilation with `P010` ([Descriptions](../models/configuration.md#descriptions)). + `Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. ## Provider name @@ -53,6 +56,7 @@ Override the name explicitly with `provider_name`: ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" provider_name = "warehouse" api_key: str ``` @@ -68,6 +72,7 @@ from sqlbuild.tasks import task @task def export_orders(ctx, warehouse_client): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -81,6 +86,7 @@ from providers.warehouse_client import WarehouseClient @task def export_orders(ctx, warehouse_client: WarehouseClient): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -103,11 +109,13 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send(f"Model {ctx.model_name} built successfully") ``` ```sql MODEL ( + description "One row per order", materialized table, post_hooks [python("notify_complete")], ); @@ -118,6 +126,7 @@ Providers are also available on the `HookContext` via `ctx.providers`: ```python @hook def notify_complete(ctx): + """Notify complete.""" notifier = ctx.providers.slack_notifier notifier.send(f"Model {ctx.model_name} built successfully") ``` @@ -129,6 +138,7 @@ All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `Loader ```python @task def export_orders(ctx): + """Export orders.""" client = ctx.providers.warehouse_client client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -145,6 +155,7 @@ Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str def setup(self, ctx): @@ -164,6 +175,7 @@ Because `Provider` extends `pydantic-settings BaseSettings`, fields without defa ```python class SlackNotifier(Provider): + """Posts build notifications to a chat channel.""" slack_token: str # reads SLACK_TOKEN from environment channel: str = "#builds" # has a default, environment variable is optional ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md index 3ed1ebdd4c..54c656c706 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/sql-references.md @@ -26,14 +26,17 @@ from sqlbuild.tasks import task @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" ... @task(depends_on=source("raw_orders")) def inspect_raw(ctx): + """Inspect raw.""" ... @task(depends_on=seed("country_codes")) def export_countries(ctx): + """Export countries.""" ... ``` @@ -53,6 +56,7 @@ Always resolve a reference to its concrete relation with `ctx.relation(...)` ins ```python @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT * FROM {relation}").fetchall() return ctx.result(metadata={"rows": len(rows)}) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md index 032ca4c17b..73d67cd81d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md @@ -28,6 +28,7 @@ from sqlbuild.tasks import task, TaskContext @task def export_orders(ctx: TaskContext): + """Export orders.""" rows = fetch_orders() return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) ``` @@ -41,10 +42,12 @@ Tasks declare dependencies with `depends_on`, accepting a single function, a tup ```python @task def fetch_orders(ctx: TaskContext): + """Fetch orders.""" return ctx.result(payload=download_orders()) @task(depends_on=fetch_orders) def summarize_orders(ctx: TaskContext): + """Summarize orders.""" result = ctx.result_of(fetch_orders) return ctx.result(metadata={"count": len(result.payload)}) ``` @@ -58,6 +61,7 @@ Tasks may depend on other tasks, assets, and loaders. They may **not** depend on ```python @task def build_export(ctx: TaskContext): + """Build export.""" return ctx.result( payload={"path": "/exports/orders.csv"}, metadata={"rows": 1200}, @@ -75,6 +79,7 @@ Return `ctx.skip(...)` to skip a task: ```python @task def export_if_present(ctx: TaskContext): + """Export if present.""" if not new_files_available(): return ctx.skip("no new files") return ctx.result(payload=do_export()) @@ -90,6 +95,7 @@ from sqlbuild.tasks import task, SkipMode @task def optional_step(ctx): + """Optional step.""" return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" ``` @@ -103,6 +109,7 @@ from sqlbuild.tasks import task @task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) def call_api(ctx): + """Call api.""" return ctx.result(payload=fetch_from_flaky_api()) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 833d151170..03964b35c9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -41,6 +41,11 @@ XSQBR001 → XSQBR Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching semantics and takes precedence over `select`. +A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired +when descriptions became the compile error `P010` +([Descriptions](../models/configuration.md#descriptions)); remove it from `select` and +`ignore`. + Use the catalogue to inspect available codes and metadata: ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md index cbc47164d4..df22b9868d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -120,7 +120,7 @@ The header is metadata only: | Field | Description | |-------|-------------| -| `description` | Optional description string | +| `description` | Required description of the behaviour the scenario proves ([Descriptions](models/configuration.md#descriptions)) | | `tags` | Optional list of string tags | ## How scenarios execute diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md index 303b4ef1d8..34e3d3f4b1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/seeds.md @@ -55,7 +55,7 @@ Seed YAML accepts only these keys; any other key is a compile error: | Seed | `name`, `description`, `database`, `schema`, `meta`, `tags`, `csv_settings`, `columns`, `audits` | | Column | `name`, `type`, `nullable`, `description`, `meta`, `audits` | -Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). +Every seed must have a YAML declaration with a non-empty `description` ([Descriptions](models/configuration.md#descriptions)) and at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). ## Referencing seeds @@ -89,6 +89,7 @@ Attach audits to a seed or its columns with the same syntax as source audits: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -111,6 +112,7 @@ Seeds inherit the project's default database and schema. You can override these ```yaml seeds: - name: waffle_types + description: Waffle types database: analytics schema: lookups columns: @@ -127,6 +129,7 @@ For non-standard CSV formats, configure parsing behavior with `csv_settings`: ```yaml seeds: - name: european_prices + description: European prices csv_settings: delimiter: ";" encoding: utf-8 diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md index 0d318ce936..b70d310a89 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/snapshots.md @@ -40,6 +40,7 @@ Use when your source has a reliable column recording when the entity changed. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -63,6 +64,7 @@ Use when the source does not have a reliable update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -91,6 +93,7 @@ Use for daily full exports or periodic snapshots without a business update times ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -114,6 +117,7 @@ Use for historical observations that include a business update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -139,6 +143,7 @@ Use for CDC tables, audit logs, or historical backfills where rows are individua ```sql MODEL ( + description "One row per order", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -244,6 +249,7 @@ The model `snapshot_full_refresh` field can only make the policy **stricter** th ```sql MODEL ( + description "Example model", materialized snapshot, ... snapshot_full_refresh deny, @@ -266,6 +272,7 @@ Snapshot models support the same audit system as other materializations. Audits ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -340,6 +347,7 @@ JOIN customer_snapshot c ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [user_id, role_id], snapshot_strategy check, @@ -357,6 +365,7 @@ FROM __source("user_role_daily") ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [product_id], snapshot_strategy timestamp, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md index 32e33b0b15..9d087b5068 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/sources.md @@ -25,6 +25,7 @@ Point at an existing table or view in your warehouse: ```yaml sources: - name: raw_events + description: Events database: analytics schema: raw table: events @@ -39,6 +40,7 @@ Define source data inline as a SQL expression. No external tables or setup scrip ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT * FROM (VALUES (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), @@ -59,6 +61,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw_orders + description: Orders columns: - name: id audits: @@ -82,6 +85,7 @@ Type enforcement is implicit for sources. If any column declares a `type`, SQLBu ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name columns: @@ -104,6 +108,7 @@ Sources can be loaded by Python functions instead of pointing at existing tables ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -135,6 +140,7 @@ Configure freshness per source with a `freshness:` block: ```yaml sources: - name: raw_events + description: Events schema: raw table: events freshness: @@ -245,7 +251,7 @@ Use [`sqb freshness`](../cli/freshness.md) to observe source freshness on demand | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | | `unique_key` | Merge key column(s) (required for `merge`) | | `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | -| `description` | Human-readable description | +| `description` | Required human-readable description ([Descriptions](models/configuration.md#descriptions)) | | `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | | `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | | `columns` | Column declarations with `name`, `type`, `nullable`, `description`, `meta`, and `audits` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md index ca5204b7bd..0fbe0b5d99 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md @@ -219,6 +219,7 @@ A source can provide a stable Dagster-native key without replacing the translato ```yaml sources: - name: raw__orders + description: Orders meta: dagster: asset_key: [warehouse, raw, orders] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 71cbda289c..851ae4f9cc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -49,6 +49,7 @@ dlt_sources: type: header_link resources: - name: raw_github_issues + description: Issues from the GitHub API write_disposition: append endpoint: path: "repos/${github_owner}/${github_repo}/issues" @@ -69,6 +70,7 @@ dlt_sources: credentials: "${postgres_connection_string}" resources: - name: raw_customers + description: Customers from the CRM database table: customers write_disposition: merge primary_key: id @@ -86,6 +88,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + description: Events from the event export files reader: csv write_disposition: append ``` @@ -97,6 +100,7 @@ dlt_sources: | Key | Description | |-----|-------------| | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | +| `description` | What the loaded rows are. Required ([Descriptions](../concepts/models/configuration.md#descriptions)). | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | @@ -148,6 +152,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_github_issues(ctx: LoaderContext): + """Github issues.""" source = rest_api_source({ "client": { "base_url": "https://api.github.com/", @@ -182,6 +187,7 @@ Bind it to a source with `managed: true` in `sources/*.yml`: # sources/github.yml sources: - name: raw_github_issues + description: Github issues managed: true table: issues columns: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index 9621233a2c..b573813aed 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -48,6 +48,7 @@ Replicate a table from PostgreSQL into your local DuckDB project: # sources/raw.yml sources: - name: raw_orders + description: Orders table: orders ingestr: source_uri: "postgresql://user:pass@host:5432/mydb" @@ -63,6 +64,7 @@ Load Stripe charges with incremental merge on a primary key: ```yaml sources: - name: raw_stripe_charges + description: Stripe charges table: charges ingestr: source_uri: "stripe://${stripe_api_key}" @@ -105,6 +107,7 @@ All string fields in the `ingestr` block support SQLBuild project variable subst ```yaml sources: - name: raw_orders + description: Orders ingestr: source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" source_table: "public.orders" diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 3451e6c91e..7a873ed3de 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -388,6 +388,7 @@ def _function_resource(function: CompiledFunction) -> dict[str, object]: "name": function.name, "relative_path": str(function.relative_path), "language": function.language.value, + "description": function.description, "return_kind": "table" if function.return_columns else "scalar", "returns": function.returns, "return_columns": [ diff --git a/src/sqlbuild/cli/commands/_helpers/playground/copy.py b/src/sqlbuild/cli/commands/_helpers/playground/copy.py index 76e3b3ca2b..cc0fd8c7c4 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/copy.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/copy.py @@ -56,6 +56,7 @@ _PYTHON_NODES_SOURCES_YML: str = """sources: - name: raw_orders + description: Raw orders loaded by the raw_orders Python loader managed: true write_strategy: table columns: @@ -70,6 +71,7 @@ """ _PYTHON_NODES_FACT_ORDERS_SQL: str = """MODEL ( + description "One row per order with its customer and amount", materialized table, columns ( order_id (nullable false, audits [not_null, unique]), diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql index 783d210d1a..d0ca81865b 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/customer_revenue.sql @@ -1,4 +1,4 @@ -MODEL (materialized table); +MODEL (description "Revenue per customer from completed waffle orders", materialized table); SELECT customer_id, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql index c7e26d3d38..c743a3cb31 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/models/fact_waffle_orders.sql @@ -1,4 +1,4 @@ -MODEL (materialized table); +MODEL (description "One row per waffle order with its customer and line total", materialized table); SELECT o.order_id, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt index 43a81a385e..dc2c376c64 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt @@ -13,6 +13,8 @@ from sqlbuild.loaders import loader ], ) def fetch_order_events(ctx): + """Fetch new order events since the last loaded sequence number.""" + if ctx.current_cursor_value is None: next_seq = 1 else: @@ -47,6 +49,8 @@ def fetch_order_events(ctx): ], ) def fetch_customers(ctx): + """Fetch customer records changed since the last load.""" + if ctx.current_cursor_value is None: return [ {"customer_id": 1, "plan_name": "basic", "load_seq": 1}, @@ -68,6 +72,8 @@ def fetch_customers(ctx): ], ) def fetch_prices(ctx): + """Fetch the current waffle price list.""" + classic_price = 600 if ctx.current_cursor_value is None else 650 return [ {"waffle_type": "classic", "price_cents": classic_price, "load_seq": 1}, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml index be3905ca72..4e545e8468 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_orders + description: Raw waffle orders loaded by the Python loader contract: enforced managed: true write_strategy: table @@ -17,6 +18,7 @@ sources: - name: load_seq type: INTEGER - name: raw_customers + description: Raw customers loaded by the Python loader contract: enforced managed: true write_strategy: table diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql index 7f1bc18a47..a2633f291e 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/table_fn__customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Orders placed by one customer with their line totals", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql index 98ef857e40..a722c20280 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/functions/sql/udf__is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml index 3d5b2d89a9..9f97ecff49 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw__customers + description: Raw customer records loaded by the Python loader contract: enforced managed: true write_strategy: table @@ -16,6 +17,7 @@ sources: type: TIMESTAMP - name: raw__orders + description: Raw order records loaded by the Python loader contract: enforced managed: true write_strategy: delete_insert @@ -35,6 +37,7 @@ sources: type: VARCHAR - name: raw__payments + description: Payment records for each order, defined inline expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index bd2634be06..fefad914c0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -88,18 +88,14 @@ cursor_intrinsics_analysis_sql, get_validated_model_cursor_intrinsics, ) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_declaration_scope_resolver, -) +from sqlbuild.compiler.compile._helpers.render.declarations import build_declaration_scope_resolver from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros, find_macro_call_names, ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys -from sqlbuild.compiler.compile._helpers.sql_tests.identity import ( - build_sql_test_case_fingerprint, -) +from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, @@ -1690,6 +1686,7 @@ def _assemble_compiled_function( entry_point=function_input.entry_point, packages=function_input.packages, tags=function_input.tags, + description=function_input.description, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 32fc8f463c..b2737a43b7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -9,6 +9,7 @@ MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY, MEASUREMENT_THRESHOLDS_HEADER_KEY, ) +from sqlbuild.compiler.auditing.main._builtins import build_builtin_audit_resolution from sqlbuild.compiler.auditing.main._parse_audit_instance import ( parse_measurement_thresholds, parse_minimum_samples, @@ -119,6 +120,32 @@ def remember_scoped_declarations( self.scoped_declarations[key] = declarations +def build_project_audit_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + model_inputs: tuple[CompileModelInput, ...], + source_inputs: tuple[CompileSourceInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], +) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: + """Resolve built-in-aware generic audits and build every project audit input.""" + + generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] + diagnostics: tuple[CompilerDiagnostic, ...] + generic_audit_definitions, diagnostics = build_builtin_audit_resolution( + index_generic_audit_definitions(discovered_inputs.audit_files) + ) + audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( + discovered_inputs=discovered_inputs, + context=context, + model_inputs=model_inputs, + source_inputs=source_inputs, + generic_audit_definitions=generic_audit_definitions, + seed_inputs=seed_inputs, + ) + return audit_inputs, diagnostics + + def build_audit_inputs( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2b273a7ae..c2260c1862 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -170,7 +170,11 @@ _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_REUSABLE_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"columns"}) +_MODEL_HEADER_COLUMNS_KEY: str = "columns" +_MODEL_HEADER_DESCRIPTION_KEY: str = "description" +_REUSABLE_MODEL_HEADER_KEYS: frozenset[str] = frozenset( + {_MODEL_HEADER_COLUMNS_KEY, _MODEL_HEADER_DESCRIPTION_KEY} +) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} ) @@ -286,6 +290,9 @@ class _ReusableModelConfigCache: path_defaults: dict[str, dict[str, object]] target_config: TargetConfig | None reusable_by_path_default: dict[str | None, bool] = field(default_factory=dict) + inherited_values_by_path_default: dict[str | None, dict[str, object]] = field( + default_factory=dict + ) reusable_metadata_by_identity: dict[int, tuple[object, bool]] = field(default_factory=dict) configs: dict[str | None, CompileModelConfig] = field(default_factory=dict) @@ -315,9 +322,15 @@ def remember( reusable_metadata: dict[str, object] | None = self._reusable_metadata(model_header_values) if reusable_metadata is None or not self._is_reusable(matched_path_default): return + inherited_values: dict[str, object] = self.inherited_values_by_path_default[ + matched_path_default + ] reusable_values: dict[str, object] = { key: value for key, value in config.values.items() if key not in reusable_metadata } + reusable_values.update( + (key, inherited_values[key]) for key in reusable_metadata if key in inherited_values + ) self.configs[matched_path_default] = replace(config, values=reusable_values) def _reusable_metadata( @@ -326,9 +339,15 @@ def _reusable_metadata( ) -> dict[str, object] | None: if not model_header_values: return {} - if model_header_values.keys() != _REUSABLE_MODEL_HEADER_KEYS: + if not model_header_values.keys() <= _REUSABLE_MODEL_HEADER_KEYS: + return None + if _contains_dynamic_or_unsupported_reusable_metadata( + model_header_values.get(_MODEL_HEADER_DESCRIPTION_KEY) + ): return None - metadata: object = model_header_values["columns"] + if _MODEL_HEADER_COLUMNS_KEY not in model_header_values: + return model_header_values + metadata: object = model_header_values[_MODEL_HEADER_COLUMNS_KEY] cached: tuple[object, bool] | None = self.reusable_metadata_by_identity.get(id(metadata)) if cached is not None and cached[0] is metadata: return model_header_values if cached[1] else None @@ -356,6 +375,7 @@ def _is_reusable(self, matched_path_default: str | None) -> bool: (layered_values, target_namespace_values) ) and not _contains_mutable_nested_config(layered_values) self.reusable_by_path_default[matched_path_default] = reusable + self.inherited_values_by_path_default[matched_path_default] = layered_values return reusable diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 014b743b52..03fc4cbf80 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -262,6 +262,11 @@ def build_sql_function_inputs( relative_path=function_file.relative_path, language="SQL", ), + description=_parse_function_description( + header_values=header_values, + relative_path=function_file.relative_path, + language="SQL", + ), declaration_usages=expansion.usages, ) ) @@ -442,6 +447,11 @@ def _build_python_function_input( relative_path=python_function_file.relative_path, language="Python", ), + description=_parse_function_description( + header_values=header_values, + relative_path=python_function_file.relative_path, + language="Python", + ), ) @@ -590,6 +600,19 @@ def _parse_python_packages(*, raw_packages: object | None, relative_path: Path) return tuple(packages) +def _parse_function_description( + *, header_values: dict[str, object], relative_path: Path, language: str +) -> str | None: + raw_description: object | None = header_values.get("description") + if raw_description is None: + return None + if not isinstance(raw_description, str): + raise CompileInputError( + f"{language} function file {relative_path} description must be a string" + ) + return raw_description.strip() + + def _parse_function_tags( *, header_values: dict[str, object], relative_path: Path, language: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 499df98900..fc5f99faaf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -53,6 +53,8 @@ ) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) +_MODEL_DESCRIPTION_KEY: str = "description" +_MODEL_SCHEMA_KEY: str = "model_schema" def _contains_template_data_cached(*, value: object, cache: IdentityPresenceCache | None) -> bool: @@ -531,6 +533,36 @@ def _resolve_model_schema( return declaration +def has_resolved_model_description( + *, + model_file: DiscoveredSqlModelFile, + defaults: DefaultsConfig, + path_defaults: dict[str, dict[str, object]], + model_schemas: dict[str, ModelSchemaDeclaration], +) -> bool: + """Return whether layered config or the model schema gives the model a description.""" + + values: dict[str, object] = build_layered_model_values( + defaults=defaults, + path_defaults=path_defaults, + matched_path_default=find_matching_path_default( + model_file=model_file, path_defaults=path_defaults + ), + model_header_values=model_file.header_values, + ) + schema_name: object | None = values.get(_MODEL_SCHEMA_KEY) + schema: ModelSchemaDeclaration | None = ( + model_schemas.get(schema_name) if isinstance(schema_name, str) else None + ) + return _is_non_empty_text(values.get(_MODEL_DESCRIPTION_KEY)) or _is_non_empty_text( + schema.description if schema is not None else None + ) + + +def _is_non_empty_text(value: object | None) -> bool: + return isinstance(value, str) and bool(value.strip()) + + def find_matching_path_default( *, model_file: DiscoveredSqlModelFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py new file mode 100644 index 0000000000..ec08682bef --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py @@ -0,0 +1,458 @@ +"""Require a non-empty description on every named project resource.""" + +from __future__ import annotations + +import ast +import inspect +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from functools import partial +from pathlib import Path + +from sqlbuild.compiler.compile.constants import MISSING_DESCRIPTION_CODE +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlFunctionInput, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, + FunctionLanguage, +) +from sqlbuild.compiler.discovery.constants import PYTHON_UDF_DECORATOR_NAME, YAML_FILE_SUFFIXES +from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line +from sqlbuild.compiler.discovery.models import ( + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + DiscoveredProvider, +) +from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment +from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.snippet_help import snippet_help +from sqlbuild.spec.contracts.models import SourceLocation + +_LINE_COMMENT_START: str = "--" +_BLOCK_COMMENT_START: str = "/*" + + +@dataclass(frozen=True) +class _Resource: + """One named resource and the description it declares, if any.""" + + kind: str + name: str + description: str | None + location: Callable[[], SourceLocation] + help: str + resource_type: CompiledResourceType | None = None + + +@dataclass(frozen=True) +class _PythonNode: + """The shared shape of a decorated Python node.""" + + kind: str + decorator: str + name: str + description: str | None + relative_path: Path + function: Callable[..., object] + + +def missing_description_diagnostics( + *, + discovered_inputs: DiscoveredProjectInputs, + model_inputs: tuple[CompileModelInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], + source_inputs: tuple[CompileSourceInput, ...], + function_inputs: tuple[CompileSqlFunctionInput, ...], +) -> tuple[CompilerDiagnostic, ...]: + """Return one error for each named resource without a non-empty description.""" + + loaders_by_name: dict[str, DiscoveredLoaderFunction] = { + loader.name: loader for loader in discovered_inputs.loader_functions + } + resources: tuple[_Resource, ...] = ( + *(_model(model_input) for model_input in model_inputs), + *_scenarios(discovered_inputs), + *(_seed(seed_input) for seed_input in seed_inputs), + *( + _source(source_input=source_input, loaders_by_name=loaders_by_name) + for source_input in source_inputs + ), + *(_function(function_input) for function_input in function_inputs), + *_sql_hooks(discovered_inputs), + *( + _python_resource(node) + for node in _python_nodes( + discovered_inputs=discovered_inputs, source_inputs=source_inputs + ) + ), + *(_provider(provider) for provider in discovered_inputs.providers), + ) + return tuple( + _diagnostic(resource) + for resource in resources + if resource.description is None or not resource.description.strip() + ) + + +def _diagnostic(resource: _Resource) -> CompilerDiagnostic: + state: str = ( + "has no description" if resource.description is None else "has an empty description" + ) + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=MISSING_DESCRIPTION_CODE, + message=f"{resource.kind} '{resource.name}' {state}", + resource_type=resource.resource_type, + resource_name=resource.name, + location=resource.location(), + help=resource.help, + ) + + +def _model(model_input: CompileModelInput) -> _Resource: + name: str = model_input.model_file.file_path.stem + path: Path = model_input.model_file.relative_path + contents: str = model_input.model_file.contents + help_text: str = _header_help( + kind="model", + name=name, + statement="MODEL", + entry=f'description "What one row of {name} represents"', + path=path, + ) + schema_entry_description: str | None = ( + model_input.schema_entry.description if model_input.schema_entry is not None else None + ) + return _Resource( + kind="model", + name=name, + description=schema_entry_description, + location=partial(_header_location, path=path, contents=contents), + help=help_text, + resource_type=CompiledResourceType.MODEL, + ) + + +def _scenarios(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: + for scenario_file in discovered_inputs.scenario_files: + raw_description: object | None = scenario_file.header_values.get("description") + yield _Resource( + kind="scenario", + name=scenario_file.name, + description=raw_description if isinstance(raw_description, str) else None, + location=partial( + _header_location, + path=scenario_file.relative_path, + contents=scenario_file.contents, + ), + help=_header_help( + kind="scenario", + name=scenario_file.name, + statement="SCENARIO", + entry=f'description "The behaviour {scenario_file.name} proves"', + path=scenario_file.relative_path, + ), + resource_type=CompiledResourceType.SQL_SCENARIO, + ) + + +def _seed(seed_input: CompileSeedInput) -> _Resource: + name: str = seed_input.schema_entry.name + return _Resource( + kind="seed", + name=name, + description=seed_input.schema_entry.description, + location=partial( + _yaml_location, + path=seed_input.schema_file.relative_path, + contents=seed_input.schema_file.contents, + name=name, + ), + help=_yaml_help(kind="seed", name=name, path=seed_input.schema_file.relative_path), + resource_type=CompiledResourceType.SEED, + ) + + +def _source( + *, source_input: CompileSourceInput, loaders_by_name: dict[str, DiscoveredLoaderFunction] +) -> _Resource: + name: str = source_input.source_entry.name + loader: DiscoveredLoaderFunction | None = ( + loaders_by_name.get(source_input.source_entry.loader) + if source_input.source_entry.loader is not None + else None + ) + description: str | None = source_input.source_entry.description + if (description is None or not description.strip()) and loader is not None: + description = loader.description or description + help_text: str = _yaml_help( + kind="source", name=name, path=source_input.source_file.relative_path + ) + if loader is not None and loader.relative_path.suffix not in YAML_FILE_SUFFIXES: + help_text = join_helps( + help_text, + f"or describe its loader '{loader.name}' in {loader.relative_path.as_posix()} " + f'with a docstring or @loader(description="What one row of {name} represents")', + ) + return _Resource( + kind="source", + name=name, + description=description, + location=partial( + _yaml_location, + path=source_input.source_file.relative_path, + contents=source_input.source_file.contents, + name=name, + ), + help=help_text, + resource_type=CompiledResourceType.SOURCE, + ) + + +def _function(function_input: CompileSqlFunctionInput) -> _Resource: + name: str = function_input.name + path: Path = function_input.function_file.relative_path + resource_type: CompiledResourceType = ( + CompiledResourceType.TABLE_FN if function_input.return_columns else CompiledResourceType.UDF + ) + location: Callable[[], SourceLocation] + help_text: str + if function_input.language == FunctionLanguage.PYTHON: + location = partial( + _decorator_location, + path=path, + contents=function_input.function_file.contents, + decorator=PYTHON_UDF_DECORATOR_NAME, + ) + help_text = snippet_help( + purpose=f"to describe function '{name}'", + target=f"give its function in {path.as_posix()} a docstring or add this to @udf", + lines=(f'@udf(description="What {name} returns", ...)',), + ) + else: + location = partial( + _header_location, path=path, contents=function_input.function_file.contents + ) + help_text = _header_help( + kind="function", + name=name, + statement="FUNCTION", + entry=f'description "What {name} returns"', + path=path, + ) + return _Resource( + kind="function", + name=name, + description=function_input.description, + location=location, + help=help_text, + resource_type=resource_type, + ) + + +def _sql_hooks(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: + for hook_file in discovered_inputs.sql_hook_files: + yield _Resource( + kind="hook", + name=hook_file.name, + description=hook_file.description, + location=partial( + _header_location, path=hook_file.relative_path, contents=hook_file.contents + ), + help=_header_help( + kind="hook", + name=hook_file.name, + statement="HOOK", + entry=f'description "What {hook_file.name} does"', + path=hook_file.relative_path, + description_only=True, + ), + ) + + +def _python_nodes( + *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[CompileSourceInput, ...] +) -> Iterator[_PythonNode]: + source_owned_loaders: frozenset[str] = frozenset( + source_input.source_entry.loader + for source_input in source_inputs + if source_input.source_entry.loader is not None + ) + source_paths: frozenset[Path] = frozenset( + source_file.relative_path for source_file in discovered_inputs.source_files + ) + for loader in discovered_inputs.loader_functions: + if loader.name in source_owned_loaders or loader.relative_path in source_paths: + continue + yield _PythonNode( + kind="loader", + decorator="loader", + name=loader.name, + description=loader.description, + relative_path=loader.relative_path, + function=loader.function, + ) + for kind, functions in ( + ("task", discovered_inputs.task_functions), + ("asset", discovered_inputs.asset_functions), + ("check", discovered_inputs.check_functions), + ("hook", discovered_inputs.hook_functions), + ): + for function in functions: + yield _PythonNode( + kind=kind, + decorator=kind, + name=function.name, + description=function.description, + relative_path=function.relative_path, + function=function.function, + ) + + +def _python_resource(node: _PythonNode) -> _Resource: + return _Resource( + kind=node.kind, + name=node.name, + description=node.description, + location=partial(_function_location, path=node.relative_path, function=node.function), + help=snippet_help( + purpose=f"to describe {node.kind} '{node.name}'", + target=( + f"give its function in {node.relative_path.as_posix()} a docstring " + f"or pass a description to @{node.decorator}" + ), + lines=(f'@{node.decorator}(description="What {node.name} does", ...)',), + ), + ) + + +def _provider(provider: DiscoveredProvider) -> _Resource: + class_name: str = provider.provider_class.__name__ + raw_doc: object | None = provider.provider_class.__dict__.get("__doc__") + return _Resource( + kind="provider", + name=provider.name, + description=inspect.cleandoc(raw_doc) if isinstance(raw_doc, str) else None, + location=partial( + _class_location, path=provider.relative_path, provider_class=provider.provider_class + ), + help=snippet_help( + purpose=f"to describe provider '{provider.name}'", + target=f"add a class docstring in {provider.relative_path.as_posix()}", + lines=( + f"class {class_name}(Provider):", + f' """What {provider.name} provides."""', + ), + ), + ) + + +def _header_help( + *, + kind: str, + name: str, + statement: str, + entry: str, + path: Path, + description_only: bool = False, +) -> str: + lines: tuple[str, ...] = ( + (f"{statement} ({entry});",) + if description_only + else (f"{statement} (", f" {entry},", " ...", ");") + ) + return snippet_help( + purpose=f"to describe {kind} '{name}'", + target=f"add this to the {statement} header in {path.as_posix()}", + lines=lines, + ) + + +def _yaml_help(*, kind: str, name: str, path: Path) -> str: + return snippet_help( + purpose=f"to describe {kind} '{name}'", + target=f"add `description` to its entry in {path.as_posix()}", + lines=(f"- name: {name}", f" description: What one row of {name} represents"), + ) + + +def _header_location(*, path: Path, contents: str) -> SourceLocation: + """Locate the statement header: the first code after leading whitespace and comments.""" + + position: int = 0 + while True: + while position < len(contents) and contents[position].isspace(): + position += 1 + if contents.startswith(_LINE_COMMENT_START, position): + position = skip_line_comment(sql=contents, start=position) + elif contents.startswith(_BLOCK_COMMENT_START, position): + position = skip_block_comment(sql=contents, start=position) + else: + return _offset_location(path=path, contents=contents, offset=position) + + +def _function_location(*, path: Path, function: Callable[..., object]) -> SourceLocation: + return SourceLocation(path=path, line=_function_line(function), column=1) + + +def _class_location(*, path: Path, provider_class: type[object]) -> SourceLocation: + return SourceLocation(path=path, line=_class_line(provider_class), column=1) + + +def _yaml_location(*, path: Path, contents: str, name: str) -> SourceLocation: + line: int | None = yaml_entry_line(contents=contents, name=name) + return SourceLocation(path=path, line=line or 1, column=1) + + +def _decorator_location(*, path: Path, contents: str, decorator: str) -> SourceLocation: + """Locate the `@decorator` line of the decorated function, ignoring docstrings and comments.""" + + try: + module: ast.Module = ast.parse(contents) + except SyntaxError: + return SourceLocation(path=path, line=1, column=1) + for node in ast.walk(module): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + for expression in node.decorator_list: + if _decorator_name(expression) == decorator: + return SourceLocation( + path=path, line=expression.lineno, column=expression.col_offset + ) + return SourceLocation(path=path, line=1, column=1) + + +def _decorator_name(expression: ast.expr) -> str | None: + target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression + if isinstance(target, ast.Name): + return target.id + if isinstance(target, ast.Attribute): + return target.attr + return None + + +def _offset_location(*, path: Path, contents: str, offset: int) -> SourceLocation: + line: int = contents.count("\n", 0, offset) + 1 + column: int = offset - (contents.rfind("\n", 0, offset) + 1) + 1 + return SourceLocation(path=path, line=line, column=column) + + +def _function_line(function: Callable[..., object]) -> int: + code: object | None = getattr(inspect.unwrap(function), "__code__", None) + line: object | None = getattr(code, "co_firstlineno", None) + return line if isinstance(line, int) else 1 + + +def _class_line(provider_class: type[object]) -> int: + try: + return inspect.getsourcelines(provider_class)[1] + except (OSError, TypeError): + return 1 diff --git a/src/sqlbuild/compiler/compile/classes/model_description_resolution.py b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py new file mode 100644 index 0000000000..5a42738641 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/model_description_resolution.py @@ -0,0 +1,41 @@ +"""Compile-equivalent description resolution for header-only tooling.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.model_config import ( + has_resolved_model_description, +) +from sqlbuild.compiler.compile._helpers.render.declarations import ( + build_public_model_schema_index, +) +from sqlbuild.compiler.discovery.main._model_description_inputs import ( + discover_model_description_inputs, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, ModelSchemaDeclaration + + +class ModelDescriptionResolution: + """Resolve model descriptions that come from outside the MODEL header.""" + + @staticmethod + def externally_described_models(*, project_dir: Path) -> frozenset[Path]: + """Return model files that `[defaults]`, `[path_defaults]` or a model schema describe.""" + + discovered_inputs: DiscoveredProjectInputs = discover_model_description_inputs( + project_dir=project_dir + ) + model_schemas: dict[str, ModelSchemaDeclaration] = build_public_model_schema_index( + discovered_inputs=discovered_inputs + ) + return frozenset( + model_file.file_path + for model_file in discovered_inputs.model_files + if has_resolved_model_description( + model_file=model_file, + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + model_schemas=model_schemas, + ) + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 73d04a6510..063b1f129f 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -15,6 +15,39 @@ HOOK_READS_OWN_DEPENDANT_CODE: str = "P007" HARD_CODED_PROJECT_RELATION_CODE: str = "P008" UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" +MISSING_DESCRIPTION_CODE: str = "P010" +DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { + "model_files": "model", + "scenario_files": "scenario", + "seed_files": "seed", + "source_files": "source", + "sql_function_files": "function", + "python_function_files": "function", + "sql_hook_files": "hook", + "hook_functions": "hook", + "loader_functions": "loader", + "task_functions": "task", + "asset_functions": "asset", + "check_functions": "check", + "providers": "provider", +} +DESCRIPTION_EXEMPT_INPUTS: dict[str, str] = { + "project_config": "project configuration, not a named resource", + "local_config": "local configuration, not a named resource", + "project_dir": "project location, not a named resource", + "adapter_file": "project adapter override, not a named resource", + "enum_files": "typed value declarations used inside SQL, not graph resources", + "constant_files": "typed value declarations used inside SQL, not graph resources", + "macro_files": "SQL helpers expanded inline, not graph resources", + "model_schema_files": "shared column schemas; models that use one are checked", + "schema_files": "seed YAML declarations; each declared seed is checked", + "test_files": "SQL unit tests are named by their subject and expected behaviour", + "audit_files": "audits are assertions named by what they check", + "audit_factories": "generate audits, which are assertions", + "materialization_files": "materialization strategies, not graph resources", + "event_exporters": "runtime event outputs, not graph resources", + "command_output_sinks": "runtime command outputs, not graph resources", +} SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" MODEL_DIRECTORY_NAME: str = "models" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 534c86793f..0ffd4eb089 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -5,11 +5,7 @@ from dataclasses import replace from pathlib import Path -from sqlbuild.compiler.auditing.main._builtins import build_builtin_audit_resolution -from sqlbuild.compiler.compile._helpers.attachment.audits import ( - build_audit_inputs, - index_generic_audit_definitions, -) +from sqlbuild.compiler.compile._helpers.attachment.audits import build_project_audit_inputs from sqlbuild.compiler.compile._helpers.attachment.core import ( build_effective_connection, build_effective_settings, @@ -34,6 +30,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( with_collected_compile_diagnostics, ) +from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( + missing_description_diagnostics, +) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -61,8 +60,6 @@ ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredAuditBlock, - DiscoveredAuditFile, DiscoveredProjectInputs, EnumDeclaration, ModelSchemaDeclaration, @@ -191,23 +188,26 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) - project_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = ( - index_generic_audit_definitions(discovered_inputs.audit_files) - ) - generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] - diagnostics: tuple[CompilerDiagnostic, ...] - generic_audit_definitions, diagnostics = build_builtin_audit_resolution( - project_audit_definitions - ) - diagnostics = (*diagnostics, *model_build.diagnostics) - audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( + audit_inputs: tuple[CompileAuditInput, ...] + audit_diagnostics: tuple[CompilerDiagnostic, ...] + audit_inputs, audit_diagnostics = build_project_audit_inputs( discovered_inputs=discovered_inputs, context=model_context, model_inputs=model_build.inputs, source_inputs=source_inputs, - generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, ) + diagnostics: tuple[CompilerDiagnostic, ...] = ( + *audit_diagnostics, + *model_build.diagnostics, + *missing_description_diagnostics( + discovered_inputs=discovered_inputs, + model_inputs=model_build.inputs, + seed_inputs=seed_inputs, + source_inputs=source_inputs, + function_inputs=sql_function_inputs, + ), + ) return CompileProjectInputs( project_config=discovered_inputs.project_config, local_config=discovered_inputs.local_config, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 282597ae36..93bfc4495e 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -738,6 +738,7 @@ class CompileSqlFunctionInput: packages: tuple[str, ...] = field(default_factory=tuple) declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) tags: tuple[str, ...] = field(default_factory=tuple) + description: str | None = None @dataclass(frozen=True) @@ -980,6 +981,7 @@ class CompiledFunction: entry_point: str | None = None packages: tuple[str, ...] = field(default_factory=tuple) tags: tuple[str, ...] = field(default_factory=tuple) + description: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index da75d7a2b3..0abb5c741f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -1658,6 +1658,7 @@ def _append_python_node_function( unique_key=loader_definition.unique_key, columns=loader_definition.columns, contract=loader_definition.contract, + description=loader_definition.description, provider_usages=_provider_usages( function=function, provider_by_name=resolved_provider_by_name, diff --git a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py index c6f1d2e12b..7a01646d0e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py +++ b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py @@ -50,7 +50,15 @@ {"config", "destination", "resources", "schema", "type"} ) _DLT_RESOURCE_COMMON_KEYS: frozenset[str] = frozenset( - {"incremental", "merge_key", "name", "primary_key", "schema", "write_disposition"} + { + "description", + "incremental", + "merge_key", + "name", + "primary_key", + "schema", + "write_disposition", + } ) _DLT_RESOURCE_TYPE_KEYS: dict[str, frozenset[str]] = { DLT_SOURCE_TYPE_FILESYSTEM: frozenset({"reader"}), @@ -338,6 +346,13 @@ def _parse_dlt_resource_entry( ) return SourceEntry( name=name, + description=optional_non_empty_string( + entry=resource, + key="description", + file_path=file_path, + label=f"dlt resource '{name}'", + error_class=SourceParseError, + ), schema=schema, table=name, managed=True, @@ -426,6 +441,7 @@ def _raw_dlt_resource_config(*, source_type: str, resource: dict[str, object]) - "merge_key", "incremental", "schema", + "description", } if source_type == DLT_SOURCE_TYPE_REST_API: return {"endpoint": resource["endpoint"]} diff --git a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py index b4814b41bf..80493330ba 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/python/functions.py +++ b/src/sqlbuild/compiler/discovery/_helpers/python/functions.py @@ -13,6 +13,8 @@ ) from sqlbuild.compiler.discovery.exceptions import ModelSqlParseError +_UDF_DESCRIPTION_KEY: str = "description" + def parse_python_function(*, contents: str, file_path: Path) -> tuple[dict[str, object], str, str]: """Parse one @udf-decorated Python function without importing project code.""" @@ -75,6 +77,9 @@ def parse_python_function(*, contents: str, file_path: Path) -> tuple[dict[str, error_class=ModelSqlParseError, ) + docstring: str | None = ast.get_docstring(function_node) + if _UDF_DESCRIPTION_KEY not in values and docstring is not None: + values[_UDF_DESCRIPTION_KEY] = docstring entry_point: object | None = values.get("entry_point") if entry_point is None: values["entry_point"] = function_node.name diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 5777e15c13..2d2396b48f 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -18,7 +18,7 @@ PYTHON_NODE_ROOT: str = "python" PYTHON_UDF_DECORATOR_NAME: str = "udf" SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( - {"arguments", "returns", "database", "schema", "tags"} + {"arguments", "returns", "database", "schema", "tags", "description"} ) SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( { @@ -94,6 +94,7 @@ REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) PYTHON_UDF_KEYS: frozenset[str] = frozenset( { + "description", "arguments", "returns", "runtime_version", diff --git a/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py new file mode 100644 index 0000000000..5925541c97 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/_model_description_inputs.py @@ -0,0 +1,37 @@ +"""SQL-only discovery of the inputs that decide a model's description.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_model_files, + discover_model_schema_files, +) +from sqlbuild.compiler.discovery._helpers.yml.project import load_project_config +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault +from sqlbuild.spec.contracts.models import LocalConfig + + +def discover_model_description_inputs(*, project_dir: Path) -> DiscoveredProjectInputs: + """Load project config, model files and model schemas without importing project Python.""" + + return DiscoveredProjectInputs( + project_config=load_project_config(project_dir=project_dir), + local_config=LocalConfig(), + model_files=discover_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=False, + extract_output_column_locations=False, + on_fault=_skip_unreadable_file, + ), + model_schema_files=discover_model_schema_files( + project_dir=project_dir, on_fault=_skip_unreadable_file + ), + ) + + +def _skip_unreadable_file(fault: DiscoveryFileFault) -> None: + """Leave an unparseable file unresolved; format reports that file's own faults.""" + + del fault diff --git a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py new file mode 100644 index 0000000000..375abe4f80 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py @@ -0,0 +1,24 @@ +"""Locate one named entry in an authored YAML declaration file.""" + +from __future__ import annotations + +import re + + +def yaml_entry_line(*, contents: str, name: str) -> int | None: + """Return the 1-based line of the outermost `- name: ` entry, if present.""" + + pattern: re.Pattern[str] = re.compile( + rf"^(?P\s*)-\s*name:\s*(?P['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$" + ) + best: tuple[int, int] | None = None + index: int + line: str + for index, line in enumerate(contents.splitlines(), start=1): + match: re.Match[str] | None = pattern.match(line) + if match is None: + continue + indent: int = len(match.group("indent")) + if best is None or indent < best[0]: + best = (indent, index) + return None if best is None else best[1] diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 4f6f0a5ab7..a43a6f99f9 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -436,6 +436,7 @@ class DiscoveredLoaderFunction: unique_key: tuple[str, ...] = field(default_factory=tuple) columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) contract: str | None = None + description: str | None = None connection_mode: LoaderConnectionMode = LoaderConnectionMode.SQLBUILD provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py index f9030a58ec..77d2a32c58 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from dataclasses import dataclass from functools import partial from typing import Any @@ -13,6 +12,7 @@ from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line from sqlbuild.compiler.planner._helpers.resolve.sources import render_source_read_relation from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope @@ -172,22 +172,5 @@ def _missing_source_tables_help(*, missing: tuple[_MissingSourceTable, ...]) -> def _declaration_location(source: CompiledSource) -> str: path: str = source.source_file.relative_path.as_posix() - line: int | None = _declaration_line(contents=source.source_file.contents, name=source.name) + line: int | None = yaml_entry_line(contents=source.source_file.contents, name=source.name) return path if line is None else f"{path}:{line}" - - -def _declaration_line(*, contents: str, name: str) -> int | None: - pattern: re.Pattern[str] = re.compile( - rf"^(?P\s*)-\s*name:\s*(?P['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$" - ) - best: tuple[int, int] | None = None - index: int - line: str - for index, line in enumerate(contents.splitlines(), start=1): - match: re.Match[str] | None = pattern.match(line) - if match is None: - continue - indent: int = len(match.group("indent")) - if best is None or indent < best[0]: - best = (indent, index) - return None if best is None else best[1] diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py index 69090a97d3..5b7b95f5ac 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py @@ -224,6 +224,7 @@ def _build_loader_node( else None, }, ), + description=loader.description, loader=DiscoveredPythonLoaderMetadata( destination=loader.destination, write_strategy=loader.write_strategy, diff --git a/src/sqlbuild/errors/setting_help/_helpers/text.py b/src/sqlbuild/errors/setting_help/_helpers/text.py index cded5445f0..3bbe56e013 100644 --- a/src/sqlbuild/errors/setting_help/_helpers/text.py +++ b/src/sqlbuild/errors/setting_help/_helpers/text.py @@ -61,5 +61,10 @@ def model_header_help(*, purpose: str, entry: str, follow_up: str | None) -> str return "\n".join(lines) +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: + snippet: str = "\n".join(f"{SETTING_SNIPPET_INDENT}{line}" for line in lines) + return f"{purpose}, {target}:\n{snippet}" + + def join_helps(helps: tuple[str, ...]) -> str: return ADDITIONAL_HELP_SEPARATOR.join(helps) diff --git a/src/sqlbuild/errors/setting_help/main/snippet_help.py b/src/sqlbuild/errors/setting_help/main/snippet_help.py new file mode 100644 index 0000000000..6f5025d24a --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/snippet_help.py @@ -0,0 +1,11 @@ +"""Public entry for help that shows an exact authored snippet to add.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import snippet_help as _snippet_help + + +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: + """Render `, :` followed by the exact lines to write, one per line.""" + + return _snippet_help(purpose=purpose, target=target, lines=lines) diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 9855d1123e..ed33796a3d 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -8,6 +8,9 @@ from dataclasses import dataclass from pathlib import Path +from sqlbuild.compiler.compile.classes.model_description_resolution import ( + ModelDescriptionResolution, +) from sqlbuild.compiler.discovery._helpers.sql.model_files import ( parse_header_values, # noqa: FFL102 - the compiler owns the canonical native header grammar prepare_model_header_tokens, @@ -32,7 +35,6 @@ VIOLATION_SEVERITY_WARNING, ) from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation -from sqlbuild.lint.types import LintSeverity _QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) _ESCAPE_CHARACTER: str = "\\" @@ -81,7 +83,6 @@ def lint_native_headers( file_path: Path, headers: tuple[HeaderSpan, ...], config: LintConfig, - description_present_severity: LintSeverity = VIOLATION_SEVERITY_FAULT, ) -> tuple[LintViolation, ...]: """Run all native header rules and return their violations.""" @@ -96,7 +97,6 @@ def lint_native_headers( header=header, config=config, line_starts=line_starts, - description_present_severity=description_present_severity, ) ) violations.extend( @@ -176,7 +176,6 @@ def _header_parse_faults( file_path=file_path, headers=headers, config=config, - description_present_severity=VIOLATION_SEVERITY_WARNING, ) if violation.code == RULE_HEADER_PARSE ) @@ -230,7 +229,6 @@ def _lint_header_values( header: HeaderSpan, config: LintConfig, line_starts: tuple[int, ...], - description_present_severity: LintSeverity, ) -> tuple[LintViolation, ...]: header_text: str = contents[header.start : header.end] try: @@ -276,8 +274,10 @@ def _lint_header_values( header=header, code=RULE_DESCRIPTION_PRESENT, message=f"{header.kind}() header requires a description", - remediation=f"Add a description to the {header.kind}() header.", - severity=description_present_severity, + remediation=( + f'Add description "" to the {header.kind}() header; ' + "every named resource must be described." + ), ) ) if header.kind in DESCRIPTION_HEADER_KINDS and isinstance(effective_description, str): @@ -698,7 +698,6 @@ def _violation_for_header_start( code: str, message: str, remediation: str, - severity: LintSeverity = VIOLATION_SEVERITY_FAULT, ) -> LintViolation: position: tuple[int, int] = _offset_to_position( offset=header.start, line_starts=_line_starts(contents) @@ -709,7 +708,7 @@ def _violation_for_header_start( column=position[1], code=code, message=message, - severity=severity, + severity=VIOLATION_SEVERITY_FAULT, engine=LINT_ENGINE_SQLBUILD, remediation=remediation, ) @@ -839,3 +838,22 @@ def _audits_key_start(*, text: str, bracket: int) -> int: while start > 0 and text[start - 1].isspace(): start -= 1 return max(0, start - len(_AUDITS_KEYWORD)) + + +def drop_resolved_description_faults( + *, violations: list[LintViolation], project_dir: Path +) -> list[LintViolation]: + """Drop `description-present` faults for models described outside their header.""" + + if not any(violation.code == RULE_DESCRIPTION_PRESENT for violation in violations): + return violations + described: frozenset[Path] = frozenset( + path.resolve() + for path in ModelDescriptionResolution.externally_described_models(project_dir=project_dir) + ) + return [ + violation + for violation in violations + if violation.code != RULE_DESCRIPTION_PRESENT + or violation.file_path.resolve() not in described + ] diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 9ae2581a3f..ebf419e223 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -37,7 +37,9 @@ IDENTIFIER_SEPARATOR_CHARACTER: str = "_" DESCRIPTION_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_MODEL, HEADER_KIND_SCENARIO}) -DESCRIPTION_REQUIRED_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_MODEL}) +DESCRIPTION_REQUIRED_HEADER_KINDS: frozenset[str] = frozenset( + {HEADER_KIND_MODEL, HEADER_KIND_SCENARIO, HEADER_KIND_FUNCTION, HEADER_KIND_HOOK} +) DEFAULT_MAX_DESCRIPTION_LINES: int = 10 DEFAULT_LINE_WIDTH: int = 100 diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index 91075f5b48..5bccae6650 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -11,6 +11,7 @@ from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import ( + drop_resolved_description_faults, format_native_headers, lint_native_headers, prepare_native_header_cache, @@ -20,7 +21,6 @@ from sqlbuild.lint._helpers.native_format import format_native_sql_bodies from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions -from sqlbuild.lint.constants import VIOLATION_SEVERITY_WARNING from sqlbuild.lint.exceptions import LintError from sqlbuild.lint.models import ( FormatChange, @@ -90,6 +90,7 @@ def run_format( ) ) violations.extend(format_faults) + violations = drop_resolved_description_faults(violations=violations, project_dir=project_dir) final_contents: dict[Path, str] = { path: updated_contents.get(path, contents) for path, contents in files.items() } @@ -214,7 +215,6 @@ def _lint_final_contents( file_path=file_path, headers=headers, config=config, - description_present_severity=VIOLATION_SEVERITY_WARNING, ) ) return violations diff --git a/src/sqlbuild/loaders.pyi b/src/sqlbuild/loaders.pyi index 1c7bd723d1..977f5e3c38 100644 --- a/src/sqlbuild/loaders.pyi +++ b/src/sqlbuild/loaders.pyi @@ -18,5 +18,6 @@ def loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[[Callable[..., object]], Callable[..., object]]: ... def get_loader_definition(function: Callable[..., object]) -> LoaderDefinition | None: ... diff --git a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py index c602b0ee3c..458112ffa8 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py @@ -23,6 +23,7 @@ def loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Mark a Python function as a SQLBuild source loader.""" @@ -36,6 +37,7 @@ def loader( unique_key=unique_key, columns=columns, contract=contract, + description=description, ) diff --git a/src/sqlbuild/python_nodes/main/_apply_loader.py b/src/sqlbuild/python_nodes/main/_apply_loader.py index c5def399e9..f2fc9f444f 100644 --- a/src/sqlbuild/python_nodes/main/_apply_loader.py +++ b/src/sqlbuild/python_nodes/main/_apply_loader.py @@ -8,6 +8,7 @@ from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_loader_dependencies +from sqlbuild.python_nodes._helpers.description_resolution import resolve_description from sqlbuild.python_nodes._helpers.loader_normalization import ( normalize_unique_key, normalize_write_strategy, @@ -28,6 +29,7 @@ def apply_loader( unique_key: str | Sequence[str] | None = None, columns: Sequence[LoaderColumnSpec | SourceColumnEntry] = (), contract: str | None = None, + description: str | None = None, ) -> Callable[..., object] | Callable[[Callable[..., object]], Callable[..., object]]: """Apply SQLBuild loader metadata to a Python function.""" @@ -44,6 +46,7 @@ def decorate(inner: Callable[..., object]) -> Callable[..., object]: unique_key=normalize_unique_key(unique_key), columns=normalize_columns(columns), contract=contract, + description=resolve_description(function=inner, description=description), ) return attach_definition( function=inner, diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index ede8bc1cae..f297bdd362 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -106,6 +106,7 @@ class LoaderDefinition: unique_key: tuple[str, ...] = () columns: tuple[SourceColumnEntry, ...] = () contract: str | None = None + description: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py index 4f2fddc868..10a496c931 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py @@ -145,7 +145,6 @@ def _rule_example(*, rule: Rule) -> str: "SQBRTEST102": "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", "SQBRTEST103": "models/staging/stg_orders.sql maps to tests/unit/staging/.", "SQBRTEST104": 'TEST (name "stg_orders__excludes_cancelled_orders");', - "SQBRTEST105": 'SCENARIO (description "Daily revenue includes successful payments");', "SQBRTEST201": "Attach not_null/unique audits to the model key.", } if rule.guidance is not None: diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql index 574a992da8..89f0316c76 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/local_only.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model local_only.", tags [sqb_only], columns (order_id (audits [not_null])), ); diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql index 54556ccfa2..866ad7fbeb 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/deprecated_orders.sql @@ -1,3 +1,3 @@ -MODEL (tags [sqb_only, deprecated]); +MODEL (description "Test model deprecated_orders.", tags [sqb_only, deprecated]); select order_id from __ref("local_only") diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql index afdb4c8278..ce18b32233 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/downstream_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model downstream_orders.", tags [nightly, finance], columns (order_id (audits [not_null])), ); diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql index bb4ca7aa11..5940f2ab52 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/event_time_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model event_time_orders.", materialized incremental, incremental_strategy delete_insert, cursor ordered_at, diff --git a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql index dde653a700..eb082857ee 100644 --- a/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql +++ b/tests/e2e/fixtures/dbt_interop/sqlbuild_project/models/marts/mart_orders.sql @@ -1,3 +1,3 @@ -MODEL (tags [marts]); +MODEL (description "Test model mart_orders.", tags [marts]); select order_id from __ref("downstream_orders") diff --git a/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py b/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py index b666d77552..00846041ce 100644 --- a/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py +++ b/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py @@ -6,6 +6,7 @@ @loader def raw_countries(ctx: object) -> list[dict[str, object]]: + """Test loader raw_countries.""" _ = ctx return [ {"country_id": 1, "country_code": "US", "country_name": "United States"}, @@ -15,6 +16,7 @@ def raw_countries(ctx: object) -> list[dict[str, object]]: @loader def raw_webhook_events(ctx: object) -> list[dict[str, object]]: + """Test loader raw_webhook_events.""" _ = ctx return [ {"event_id": 101, "event_name": "signup"}, @@ -24,6 +26,7 @@ def raw_webhook_events(ctx: object) -> list[dict[str, object]]: @loader def raw_order_events(ctx: Any) -> list[dict[str, object]]: + """Test loader raw_order_events.""" if ctx.current_cursor_value is None: return [ { @@ -53,6 +56,7 @@ def raw_order_events(ctx: Any) -> list[dict[str, object]]: @loader def raw_customers(ctx: Any) -> list[dict[str, object]]: + """Test loader raw_customers.""" if ctx.current_cursor_value is None: return [ { @@ -82,6 +86,7 @@ def raw_customers(ctx: Any) -> list[dict[str, object]]: @loader def raw_loader_status(ctx: Any) -> None: + """Test loader raw_loader_status.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " diff --git a/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml b/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml index 95489d83b3..e8936f12cf 100644 --- a/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml +++ b/tests/e2e/fixtures/source_loader_strategies/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_countries + description: Test source raw_countries. managed: true write_strategy: table columns: @@ -11,6 +12,7 @@ sources: type: VARCHAR - name: raw_webhook_events + description: Test source raw_webhook_events. managed: true write_strategy: append columns: @@ -20,6 +22,7 @@ sources: type: VARCHAR - name: raw_order_events + description: Test source raw_order_events. managed: true write_strategy: delete_insert cursor_column: event_at @@ -32,6 +35,7 @@ sources: type: INTEGER - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: merge unique_key: customer_id @@ -45,6 +49,7 @@ sources: type: TIMESTAMP - name: raw_loader_status + description: Test source raw_loader_status. managed: true columns: - name: status_id diff --git a/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py b/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py index 219b6f23f1..c8e9572932 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py +++ b/tests/e2e/fixtures/waffle_shop/functions/python/is_completed_order_py.py @@ -7,4 +7,5 @@ runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Test function main.""" return order_status == "completed" diff --git a/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql b/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql index 7f1bc18a47..c6c3d4e447 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql +++ b/tests/e2e/fixtures/waffle_shop/functions/sql/customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql b/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql index 98ef857e40..c9af25b3bf 100644 --- a/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql +++ b/tests/e2e/fixtures/waffle_shop/functions/sql/is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Test function is_completed_order.", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py b/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py index a3c115f2ad..5c240440ca 100644 --- a/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py +++ b/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py @@ -57,12 +57,14 @@ @loader def raw_customers(ctx: object) -> list[dict[str, object]]: + """Test loader raw_customers.""" _ = ctx return CUSTOMERS @loader def raw_orders(ctx: object) -> Iterator[dict[str, object]]: + """Test loader raw_orders.""" _ = ctx for order_id, customer_id, waffle_type_id, quantity, ordered_at, status in ORDERS: yield { diff --git a/tests/e2e/fixtures/waffle_shop/sources/raw.yml b/tests/e2e/fixtures/waffle_shop/sources/raw.yml index 15d8b68fbe..702c0ee449 100644 --- a/tests/e2e/fixtures/waffle_shop/sources/raw.yml +++ b/tests/e2e/fixtures/waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -15,6 +16,7 @@ sources: type: TIMESTAMP - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: delete_insert cursor_column: ordered_at @@ -33,6 +35,7 @@ sources: type: VARCHAR - name: raw_payments + description: Test source raw_payments. expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status diff --git a/tests/e2e/scripts/compile_performance_ratio/_test_types.py b/tests/e2e/scripts/compile_performance_ratio/_test_types.py index a58f84cf90..8f517caae3 100644 --- a/tests/e2e/scripts/compile_performance_ratio/_test_types.py +++ b/tests/e2e/scripts/compile_performance_ratio/_test_types.py @@ -13,3 +13,13 @@ class CompilePerformanceRatioTestCase: max_ratio: str expected_return_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PerSideCompilePerformanceRatioTestCase: + description: str + base_generator: str + expected_return_code: int + expected_fragments: tuple[str, ...] + expected_base_projects: tuple[str, ...] + expected_head_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/compile_performance_ratio/helpers.py b/tests/e2e/scripts/compile_performance_ratio/helpers.py new file mode 100644 index 0000000000..7a3b7f1acf --- /dev/null +++ b/tests/e2e/scripts/compile_performance_ratio/helpers.py @@ -0,0 +1,61 @@ +"""Fixtures for per-side compile performance ratio comparisons.""" + +from __future__ import annotations + +import stat +import sys +from pathlib import Path + +MARKER_GENERATOR: str = """from pathlib import Path + + +def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None: + del models + (project_dir / "models").mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text('name = "orders"\\nadapter = "duckdb"\\n') + (project_dir / "models" / "base_marker.sql").write_text( + 'MODEL (description "Base marker");\\n\\nSELECT 1 AS id\\n' + ) + (project_dir / "base_marker.txt").write_text(kind) +""" +FAILING_GENERATOR: str = """def write_benchmark_project(*, kind, project_dir, models): + raise RuntimeError("base generator exploded") +""" +_LOGGING_PYTHON: str = """#!/usr/bin/env bash +project="" +previous="" +for argument in "$@"; do + if [ "$previous" = "--project-dir" ]; then project="$argument"; fi + previous="$argument" +done +if [ -n "$project" ]; then + if [ -f "$project/base_marker.txt" ]; then echo base >> "{log}"; else echo head >> "{log}"; fi +fi +exec "{python}" "$@" +""" + + +def write_fake_base_root(*, root: Path, generator: str) -> Path: + """Write a base checkout whose only content is a benchmark generator module.""" + + helpers: Path = root / "scripts" / "compile_performance_ratio" / "_helpers" + helpers.mkdir(parents=True) + for package in (root / "scripts", helpers.parent, helpers): + (package / "__init__.py").write_text("", encoding="utf-8") + (helpers / "measure.py").write_text(generator, encoding="utf-8") + return root + + +def write_logging_python(*, path: Path, log: Path) -> Path: + """Write an interpreter wrapper logging which project each compile received.""" + + path.write_text(_LOGGING_PYTHON.format(log=log, python=sys.executable), encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + log.touch() + return path + + +def logged_projects(log: Path) -> tuple[str, ...]: + """Return the project kind each logged compile received, in order.""" + + return tuple(log.read_text(encoding="utf-8").split()) diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py index e322a59273..407b597179 100644 --- a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -11,6 +11,14 @@ from tests.e2e.scripts.compile_performance_ratio._test_types import ( CompilePerformanceRatioTestCase, + PerSideCompilePerformanceRatioTestCase, +) +from tests.e2e.scripts.compile_performance_ratio.helpers import ( + FAILING_GENERATOR, + MARKER_GENERATOR, + logged_projects, + write_fake_base_root, + write_logging_python, ) _REPO_ROOT: Path = Path(__file__).resolve().parents[4] @@ -74,5 +82,74 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio assert fragment in output +@pytest.mark.parametrize( + "test_case", + ( + PerSideCompilePerformanceRatioTestCase( + description="each build compiles the project its own generator wrote", + base_generator=MARKER_GENERATOR, + expected_return_code=0, + expected_fragments=("Projects generated per side",), + expected_base_projects=("base", "base"), + expected_head_projects=("head", "head"), + ), + PerSideCompilePerformanceRatioTestCase( + description="a failing base generator stops before any compile", + base_generator=FAILING_GENERATOR, + expected_return_code=1, + expected_fragments=("base project generation in", "base generator exploded"), + expected_base_projects=(), + expected_head_projects=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_base_root_when_comparing_compile_performance_then_each_side_uses_its_project( + test_case: PerSideCompilePerformanceRatioTestCase, tmp_path: Path +) -> None: + base_root: Path = write_fake_base_root( + root=tmp_path / "base", generator=test_case.base_generator + ) + base_log: Path = tmp_path / "base.log" + head_log: Path = tmp_path / "head.log" + base_python: Path = write_logging_python(path=tmp_path / "base-python", log=base_log) + head_python: Path = write_logging_python(path=tmp_path / "head-python", log=head_log) + environment: dict[str, str] = dict(os.environ) + _ = environment.pop("GITHUB_STEP_SUMMARY", None) + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-m", + "scripts.compare_compile_performance", + "--kind", + "dense", + "--models", + "20", + "--runs", + "1", + "--base-root", + str(base_root), + "--base-python", + str(base_python), + "--head-python", + str(head_python), + "--max-ratio", + "1000", + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + env=environment, + check=False, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert logged_projects(base_log) == test_case.expected_base_projects + assert logged_projects(head_log) == test_case.expected_head_projects + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py index 9741150667..d44567130e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py @@ -34,7 +34,7 @@ def delivery_model_files() -> dict[str, str]: return { "models/customers.sql": dedent( """ - MODEL ( + MODEL (description "Test model customers.", materialized table, columns ( customer_id (audits [not_null]), @@ -45,7 +45,7 @@ def delivery_model_files() -> dict[str, str]: """ ).lstrip(), "models/orders.sql": ( - "MODEL (\n materialized table,\n columns (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n columns (\n" f"{columns}\n" " ),\n);\n\n" f"SELECT\n customer_id,\n {selected}\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py index 2619ba35ad..dd37a3407e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py @@ -258,7 +258,7 @@ def prepare_bigquery_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: models_dir: Path = project_dir / "models" / "staging" models_dir.mkdir(parents=True, exist_ok=True) (models_dir / "stg_orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model stg_orders.',\n" " materialized table,\n" " unique_key [order_id],\n" " cursor order_id,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index 35065422c2..ede85b5f93 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -172,14 +172,14 @@ def test_given_python_result_when_running_check_on_bigquery_then_persists_node_r "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -253,12 +253,12 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {dataset_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -350,7 +350,7 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro BigQueryScenarioLocalReplayE2ETestCase( description="captures bigquery fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " TIMESTAMP_TRUNC(event_ts, DAY) AS event_day,\n" @@ -360,7 +360,7 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro "GROUP BY customer_id, TIMESTAMP_TRUNC(event_ts, DAY)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TIMESTAMP '2026-01-01 08:15:00 UTC' " @@ -391,12 +391,12 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro description="reports bigquery local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, SAFE_CAST(amount_text AS INT64) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, '1500' AS amount_text\n" @@ -419,14 +419,14 @@ def test_given_bigquery_local_config_when_running_query_then_outputs_expected_ro description="reports bigquery local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(SAFE_CAST(amount_text AS INT64)) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, '1500' AS amount_text\n" @@ -535,7 +535,7 @@ def test_given_source_deferral_env_when_building_on_bigquery_then_reads_prod_and "sqlbuild_local.toml": build_bigquery_local_config(location=location), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -548,10 +548,10 @@ def test_given_source_deferral_env_when_building_on_bigquery_then_reads_prod_and "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -1111,7 +1111,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_bigquery_th repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1125,7 +1125,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_bigquery_th "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -1191,7 +1191,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1206,10 +1206,10 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1266,7 +1266,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1275,7 +1275,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1301,7 +1301,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1312,7 +1312,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1333,7 +1333,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1344,7 +1344,7 @@ def test_given_chained_loader_project_when_loading_on_bigquery_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1369,7 +1369,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_bigquery_then repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1591,7 +1591,7 @@ def test_given_bigquery_invalid_model_when_building_then_underlying_error_is_pre project_dir, dataset_name = prepare_bigquery_waffle_shop(tmp_path=tmp_path) broken_model: Path = project_dir / "models" / "marts" / "bigquery_broken_model.sql" broken_model.write_text( - "MODEL (materialized table);\n\nSELECT missing_column FROM UNNEST([STRUCT(1 AS id)])", + "MODEL (description 'Test model bigquery_broken_model.', materialized table);\n\nSELECT missing_column FROM UNNEST([STRUCT(1 AS id)])", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py index e141f10e4c..e900cbb35c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_dlt_bigquery.py @@ -66,11 +66,12 @@ def test_given_dlt_source_when_loading_to_bigquery_then_table_is_materialized_an f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 095d87af2d..f47dc5c487 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1190,7 +1190,10 @@ class PythonNodeSelectionE2ETestCase: expected_row_counts: dict[str, int] expected_node_counts: dict[str, int] expected_missing_tables: tuple[str, ...] = () - orders_sql: str = 'MODEL (materialized table);\nSELECT order_id FROM __source("raw_orders")\n' + orders_sql: str = ( + 'MODEL (description "Test model.", ' + 'materialized table);\nSELECT order_id FROM __source("raw_orders")\n' + ) existing_sql: str = "" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 60c5479ca0..5347d40f26 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -55,6 +55,7 @@ def capped_microbatch_project_files( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -62,7 +63,7 @@ def capped_microbatch_project_files( + "\n", "models/capped_events.sql": dedent( f""" - MODEL ( + MODEL (description "Test model capped_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -90,7 +91,7 @@ def capped_microbatch_project_files( + "\n", "models/downstream_events.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model downstream_events.", materialized view); SELECT id, event_time FROM __ref("capped_events") """ @@ -106,7 +107,7 @@ def capped_watermark_consumer_project_files(*, limit_action: str) -> dict[str, s repo_files["models/downstream_events.sql"] = ( dedent( """ - MODEL ( + MODEL (description "Test model downstream_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -138,7 +139,7 @@ def capped_filter_join_consumer_project_files(*, limit_action: str) -> dict[str, repo_files["models/downstream_events.sql"] = ( dedent( """ - MODEL ( + MODEL (description "Test model downstream_events.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -285,7 +286,7 @@ def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, columns (order_id (audits [not_null])), ); @@ -321,6 +322,7 @@ def build_freshness_error_branch_source_yml( f""" sources: - name: raw_orders + description: Test source raw_orders. expression: SELECT {order_id} AS order_id freshness: strategy: sql @@ -329,6 +331,7 @@ def build_freshness_error_branch_source_yml( age_policy: error_after: 1h - name: raw_customers + description: Test source raw_customers. expression: SELECT {customer_id} AS customer_id freshness: strategy: sql @@ -348,7 +351,7 @@ def replay_microbatch_model_sql(*, value_expression: str, replay_policy: str = " return ( dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -392,7 +395,10 @@ def direct_microbatch_project_toml( def raw_events_source_yml() -> str: """Return the canonical raw-events source declaration.""" - return "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + return ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" + ) def timestamp_microbatch_model_sql( @@ -407,7 +413,7 @@ def timestamp_microbatch_model_sql( return ( dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -469,10 +475,11 @@ def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, '[connection]\ndatabase = "dropped_orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" f" incremental_strategy {incremental_strategy},\n" " unique_key id,\n" @@ -496,30 +503,36 @@ def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, _GATE_TARGET_FILES: dict[str, dict[str, str]] = { "model": { "models/orders.sql": ( - "MODEL (materialized table, audits [code_check (severity error)]);\n\n" + "MODEL (description 'Test model orders.', " + "materialized table, audits [code_check (severity error)]);\n\n" "SELECT '{code}' AS code\n" ), "models/order_summary.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __ref("orders")\n' + 'MODEL (description "Test model order_summary.", ' + 'materialized table);\n\nSELECT code FROM __ref("orders")\n' ), }, "source": { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" " audits:\n - code_check:\n severity: error\n" ), "models/staged_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __source("raw_orders")\n' + 'MODEL (description "Test model staged_orders.", ' + 'materialized table);\n\nSELECT code FROM __source("raw_orders")\n' ), }, "seed": { "seeds/order_codes.yml": ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n" + " columns:\n - name: code\n" " type: VARCHAR\n audits:\n - code_check:\n severity: error\n" ), "seeds/order_codes.csv": "code\n{code}\n", "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", ' + 'materialized table);\n\nSELECT code FROM __seed("order_codes")\n' ), }, } @@ -542,7 +555,8 @@ def prepare_attached_audit_gate_project( ), "audits/generic/code_check.sql": _GATE_AUDIT_TEMPLATE.format(read=read), "models/valid_codes.sql": ( - "MODEL (materialized table);\n\nSELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" + "MODEL (description 'Test model valid_codes.', " + "materialized table);\n\nSELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" ), } files.update( @@ -575,7 +589,8 @@ def prepare_nested_source_gate_project(*, tmp_path: Path, raw_code: str) -> Path f'[connection]\ndatabase = "{ATTACHED_AUDIT_GATE_DATABASE}"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_codes\n schema: main\n table: raw_codes\n" + "sources:\n - name: raw_codes\n description: Test source raw_codes.\n" + " schema: main\n table: raw_codes\n" " audits:\n - source_check:\n severity: error\n" ), "audits/generic/source_check.sql": _GATE_AUDIT_TEMPLATE.format( @@ -585,11 +600,12 @@ def prepare_nested_source_gate_project(*, tmp_path: Path, raw_code: str) -> Path read='__source("raw_codes")' ), "models/orders.sql": ( - "MODEL (materialized table, audits [order_check (severity error)]);\n\n" + "MODEL (description 'Test model orders.', " + "materialized table, audits [order_check (severity error)]);\n\n" "SELECT 'A' AS code\n" ), "models/valid_codes.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model valid_codes.', materialized table);\n\n" "SELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" ), }, @@ -623,29 +639,35 @@ def build_asset_names(stdout: str) -> tuple[str, ...]: _AUDIT_ERROR_TARGET_FILES: dict[str, dict[str, str]] = { "source": { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" " audits:\n - broken_check:\n severity: {severity}\n" ), "audits/generic/broken_check.sql": _BROKEN_AUDIT, "models/staged_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __source("raw_orders")\n' + 'MODEL (description "Test model staged_orders.", ' + 'materialized table);\n\nSELECT code FROM __source("raw_orders")\n' ), }, "seed": { "seeds/order_codes.yml": ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n" + " columns:\n - name: code\n" " type: VARCHAR\n audits:\n - broken_check:\n" " severity: {severity}\n" ), "seeds/order_codes.csv": "code\nA\n", "audits/generic/broken_check.sql": _BROKEN_AUDIT, "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT code FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", ' + 'materialized table);\n\nSELECT code FROM __seed("order_codes")\n' ), }, "end": { - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 'A' AS code\n", - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 'A' AS code\n", + "models/orders.sql": "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT 'A' AS code\n", + "models/customers.sql": "MODEL (description 'Test model customers.', " + "materialized table);\n\nSELECT 'A' AS code\n", "audits/singular/broken_check.sql": ( "AUDIT (severity {severity});\n\n" 'SELECT o.code FROM __ref("orders") o JOIN __ref("customers") c USING (code)\n' @@ -704,12 +726,14 @@ def prepare_audit_read_plan_project(*, tmp_path: Path) -> Path: '[connection]\ndatabase = "plan_shop.duckdb"\n' ), "seeds/waffle_types.yml": ( - "seeds:\n - name: waffle_types\n columns:\n" + "seeds:\n - name: waffle_types\n description: Test seed waffle_types.\n" + " columns:\n" " - name: waffle_type_id\n type: INTEGER\n" ), "seeds/waffle_types.csv": "waffle_type_id\n1\n", "models/stg_orders.sql": ( - "MODEL (\n materialized table,\n columns (\n waffle_type_id (\n" + "MODEL (description 'Test model stg_orders.'," + "\n materialized table,\n columns (\n waffle_type_id (\n" ' audits [relationships (to __seed("waffle_types"), field waffle_type_id)],\n' " ),\n ),\n);\n\nSELECT 1 AS waffle_type_id\n" ), @@ -731,7 +755,7 @@ def prepare_audit_read_plan_project(*, tmp_path: Path) -> Path: "from sqlbuild.refs import model\n\n\n" '@hook(reads=model("stg_customers"))\n' "def record_customers(ctx):\n" - ' customers = ctx.relation(model("stg_customers"))\n' + ' """Test hook record_customers."""\n customers = ctx.relation(model("stg_customers"))\n' " ctx.execute_sql(\n" ' f"CREATE OR REPLACE TABLE customer_counts AS SELECT count(*) AS n FROM {customers}"\n' " )\n" @@ -750,20 +774,24 @@ def explicit_reference_project_files( '[connection]\ndatabase = "warehouse.duckdb"\n' + references ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": ( - "MODEL (materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" + "MODEL (description 'Test model stg_orders_eu.', " + "materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": ( - "MODEL (materialized table);\nSELECT 2 AS order_id, 20 AS amount\n" + "MODEL (description 'Test model stg_orders_us.', " + "materialized table);\nSELECT 2 AS order_id, 20 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_customers.sql": ( - "MODEL (materialized table);\nSELECT 1 AS customer_id\n" + "MODEL (description 'Test model stg_customers.', " + "materialized table);\nSELECT 1 AS customer_id\n" ), f"{EXPLICIT_REFERENCE_MODELS}/all_orders.sql": ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model all_orders.', materialized table);\n" '@union_all([__ref("stg_orders_eu"), __ref("stg_orders_us")])\n' ), f"{EXPLICIT_REFERENCE_MODELS}/orders_summary.sql": ( - 'MODEL (materialized table, post_hooks [python("record_customers")]);\n' + 'MODEL (description "Test model orders_summary.", ' + 'materialized table, post_hooks [python("record_customers")]);\n' 'SELECT count(*) AS order_count FROM __ref("all_orders")\n' ), EXPLICIT_REFERENCE_MACRO_PATH: EXPLICIT_REFERENCE_MACROS, @@ -778,10 +806,10 @@ def explicit_reference_literal_loader(*, table: str) -> str: "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" "@loader\n" "def raw_customers(ctx):\n" - f' ctx.query("SELECT count(*) FROM {table}")\n' + f' """Test loader raw_customers."""\n ctx.query("SELECT count(*) FROM {table}")\n' " return [{'id': 1}]\n" ) @@ -793,6 +821,7 @@ def _counting_python_node(*, decorator: str, name: str, ref: str, returned: str) return ( f"@{decorator}(depends_on={ref})\n" f"def {name}(ctx):\n" + f' """Test {decorator} {name}."""\n' f" relation = ctx.relation({ref})\n" f' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.{name}_result AS ' f'SELECT count(*) AS n FROM {{relation}}")\n' @@ -828,13 +857,17 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_customers\n description: Test source raw_customers.\n" + " managed: true\n write_strategy: table\n" " columns:\n - name: customer_id\n type: INTEGER\n" ), "python/loaders/customers.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_customers(ctx):\n return [{'customer_id': 99}]\n" + "@loader\ndef raw_customers(ctx):\n" + " '''Test loader raw_customers.'''\n" + " return [{'customer_id': 99}]\n" ), "python/tasks/counts.py": ( "from sqlbuild.refs import model, source\nfrom sqlbuild.tasks import task\n\n\n" @@ -846,6 +879,7 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: ), "models/orders.sql": orders_sql, "models/order_summary.sql": ( - 'MODEL (materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' + 'MODEL (description "Test model order_summary.", ' + 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index d9235b499f..78727ef17e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -73,6 +73,7 @@ def test_given_project_with_python_hooks_when_running_run_then_hooks_execute( @hook def log_hook(ctx: HookContext): + '''Test hook log_hook.''' ctx.execute_sql( f"CREATE TABLE IF NOT EXISTS {ctx.destination.schema}.hook_log " "(model_name VARCHAR, phase VARCHAR)" @@ -86,7 +87,7 @@ def log_hook(ctx: HookContext): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -202,7 +203,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -210,7 +211,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/factories/generated.py": """ from pathlib import Path @@ -228,21 +229,25 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ def generated_pipeline(): @task(name="prepare_orders", tags=("factory", "runtime")) def prepare(ctx): + '''Test task prepare.''' PROJECT_DIR.joinpath("prepare_marker.txt").write_text("ran", encoding="utf-8") return ctx.result(payload={"rows": 1}) @loader(name="raw_orders", depends_on=[prepare]) def load(ctx): + '''Test loader load.''' PROJECT_DIR.joinpath("loader_marker.txt").write_text("ran", encoding="utf-8") return [{"order_id": 1}] @asset(name="orders_export", depends_on=prepare, tags=("factory", "runtime")) def export(ctx): + '''Test asset export.''' PROJECT_DIR.joinpath("asset_marker.txt").write_text("ran", encoding="utf-8") return ctx.result(materialized=True) @check(name="orders_export_check", depends_on=export, tags=("factory", "quality")) def export_check(ctx): + '''Test check export_check.''' return ctx.pass_(message="generated export exists") return [prepare, load, export, export_check] @@ -365,14 +370,14 @@ def test_given_existing_intermediate_target_when_running_source_only_then_reuses " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) setup_result: subprocess.CompletedProcess[str] = run_sqb( @@ -426,7 +431,7 @@ def test_given_task_selector_when_running_run_then_task_executes( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" + " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'})\n" ), }, @@ -481,7 +486,7 @@ def test_given_asset_selector_when_running_run_then_asset_executes( "from sqlbuild.assets import asset\n\n" "@asset\n" "def prepared_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('asset_marker.txt').write_text('ran')\n" + " '''Test asset prepared_orders.'''\n Path(__file__).parents[2].joinpath('asset_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'}, materialized=True)\n" ), }, @@ -536,7 +541,7 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result(metadata={'rows': 3})\n" + " '''Test task prepare_orders.'''\n return ctx.result(metadata={'rows': 3})\n" ), }, ) @@ -606,7 +611,7 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( "from sqlbuild.tasks import task\n\n" "@task\n" "def fail_orders(ctx):\n" - " raise RuntimeError('API unavailable')\n" + " '''Test task fail_orders.'''\n raise RuntimeError('API unavailable')\n" ), }, ) @@ -675,7 +680,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" + " '''Test task prepare_orders.'''\n Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" " return ctx.result(metadata={'prepared': True})\n" ), "python/loaders/orders.py": ( @@ -684,14 +689,14 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from python.tasks.orders import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('orders_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('orders_ready.txt')\n" " if not marker.exists():\n" " raise RuntimeError('orders were not prepared')\n" " return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -701,7 +706,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ " type: INTEGER\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ), }, @@ -750,7 +755,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 100 AS amount_cents\n" " columns:\n" " - name: order_id\n" @@ -759,7 +764,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built " type: INTEGER\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ), "python/tasks/orders.py": ( @@ -768,7 +773,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def summarize_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test task summarize_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('summary.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" @@ -825,14 +830,14 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -840,7 +845,7 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads "from sqlbuild.refs import model\n\n" "@asset(depends_on=model('fact_orders'))\n" "def export_fact_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test asset export_fact_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('export.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" @@ -899,7 +904,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" - " return ctx.result(payload={'rows': 1})\n" + " '''Test task fetch_orders.'''\n return ctx.result(payload={'rows': 1})\n" ), "python/tasks/notify_orders.py": ( "from pathlib import Path\n" @@ -907,7 +912,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from sqlbuild.tasks import task\n\n" "@task(depends_on=publish_orders)\n" "def notify_orders(ctx):\n" - " metadata = ctx.result_of(node_function=publish_orders).metadata\n" + " '''Test task notify_orders.'''\n metadata = ctx.result_of(node_function=publish_orders).metadata\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(metadata['published']))\n" " return ctx.result()\n" @@ -917,7 +922,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "from python.tasks.fetch_orders import fetch_orders\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" - " payload = ctx.result_of(node_function=fetch_orders).payload\n" + " '''Test asset publish_orders.'''\n payload = ctx.result_of(node_function=fetch_orders).payload\n" " return ctx.result(payload=payload, metadata={'published': True})\n" ), }, @@ -964,11 +969,11 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -982,7 +987,7 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def summarize_loaded_orders(ctx):\n" - " orders = ctx.relation(source('raw_orders'))\n" + " '''Test task summarize_loaded_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), @@ -991,7 +996,7 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "from python.tasks.orders import summarize_loaded_orders\n\n" "@asset(depends_on=summarize_loaded_orders)\n" "def publish_loaded_orders(ctx):\n" - " payload = ctx.result_of(node_function=summarize_loaded_orders).payload\n" + " '''Test asset publish_loaded_orders.'''\n payload = ctx.result_of(node_function=summarize_loaded_orders).payload\n" " return ctx.result(payload=payload, materialized=False)\n" ), }, @@ -1051,13 +1056,13 @@ def test_given_skip_and_asset_selection_with_json_when_running_run_then_json_rec "from sqlbuild.tasks import task\n\n" "@task\n" "def optional_orders(ctx):\n" - " return ctx.skip(reason='no files')\n\n" + " '''Test task optional_orders.'''\n return ctx.skip(reason='no files')\n\n" ), "python/assets/orders.py": ( "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" - " return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" + " '''Test asset observed_orders.'''\n return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" ), }, ) @@ -1126,25 +1131,25 @@ def test_given_independent_python_and_sql_selectors_when_running_run_then_all_br ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __source("raw_orders")\n' ), "python/tasks/branches.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def branch_a(ctx):\n" - " Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" + " '''Test task branch_a.'''\n Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" " return ctx.result()\n\n" "@task\n" "def branch_b(ctx):\n" - " Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" + " '''Test task branch_b.'''\n Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" " return ctx.result()\n" ), }, @@ -1198,7 +1203,7 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 100 AS amount_cents\n" " columns:\n" " - name: order_id\n" @@ -1207,17 +1212,17 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def profile_stg_orders(ctx):\n" - " orders = ctx.relation(model('stg_orders'))\n" + " '''Test task profile_stg_orders.'''\n orders = ctx.relation(model('stg_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows})\n" ), @@ -1275,11 +1280,11 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1293,7 +1298,7 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" - " orders = ctx.relation(source('raw_orders'))\n" + " '''Test task profile_raw_orders.'''\n orders = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows})\n" ), @@ -1349,24 +1354,24 @@ def test_given_sql_ready_task_fails_when_running_run_then_footer_json_and_exit_f ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def fail_after_stg_orders(ctx):\n" - " raise RuntimeError('profile failed')\n" + " '''Test task fail_after_stg_orders.'''\n raise RuntimeError('profile failed')\n" ), }, ) @@ -1431,11 +1436,11 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " columns:\n" " - name: order_id\n" @@ -1446,7 +1451,7 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej "from sqlbuild.tasks import task\n\n" "@task(depends_on=raw_orders)\n" "def summarize_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task summarize_orders.'''\n return ctx.result()\n" ), }, ) @@ -1506,14 +1511,14 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo " ],\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1, 'amount_cents': 100}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1, 'amount_cents': 100}]\n" ), "python/tasks/orders.py": ( "from python.loaders.orders import stage_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=stage_orders)\n" "def summarize_stage_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" + " '''Test task summarize_stage_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), "python/assets/orders.py": ( @@ -1521,7 +1526,7 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo "from python.tasks.orders import summarize_stage_orders\n\n" "@asset(depends_on=summarize_stage_orders)\n" "def publish_stage_orders(ctx):\n" - " payload = ctx.result_of(node_function=summarize_stage_orders).payload\n" + " '''Test asset publish_stage_orders.'''\n payload = ctx.result_of(node_function=summarize_stage_orders).payload\n" " return ctx.result(payload=payload, materialized=True)\n" ), }, @@ -1584,7 +1589,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/tasks/orders.py": ( "from pathlib import Path\n" @@ -1592,7 +1597,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_raw_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test task prepare_raw_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" ), @@ -1602,14 +1607,14 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_raw_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('window orders were not prepared')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1617,7 +1622,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1666,7 +1671,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" - " return ctx.result(payload={'order_id': 1})\n" + " '''Test task fetch_orders.'''\n return ctx.result(payload={'order_id': 1})\n" ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -1674,7 +1679,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.assets import asset\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" - " payload = ctx.result_of(node_function=fetch_orders).payload\n" + " '''Test asset publish_orders.'''\n payload = ctx.result_of(node_function=fetch_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1685,14 +1690,14 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset was not ready')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1700,7 +1705,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1753,7 +1758,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/assets/orders.py": ( "from pathlib import Path\n" @@ -1761,7 +1766,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "from sqlbuild.assets import asset\n\n" "@asset(depends_on=load_window_orders)\n" "def prepare_asset_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test asset prepare_asset_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " Path(__file__).parents[2].joinpath('asset_ready.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" ), @@ -1771,14 +1776,14 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_asset_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset orders were not prepared')\n" " return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1786,7 +1791,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1839,19 +1844,19 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 1}]\n" ), "python/loaders/raw.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(load_window_orders,))\n" "def raw_orders(ctx):\n" - " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" + " '''Test loader raw_orders.'''\n rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" " return [{'order_id': rows}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1859,7 +1864,7 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -1912,14 +1917,14 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 7}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 7}]\n" ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" - " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" + " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), "python/assets/prepare.py": ( @@ -1928,7 +1933,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" - " payload = ctx.result_of(node_function=prepare_orders).payload\n" + " '''Test asset publish_prepared_orders.'''\n payload = ctx.result_of(node_function=prepare_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1939,12 +1944,12 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1952,14 +1957,14 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" - " orders = ctx.relation(model('fact_orders'))\n" + " '''Test task profile_fact_orders.'''\n orders = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {orders}').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), @@ -1968,7 +1973,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" + " '''Test asset export_fact_orders.'''\n payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), "python/tasks/notify.py": ( @@ -1977,7 +1982,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=export_fact_orders).payload\n" + " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" @@ -2050,7 +2055,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" @@ -2061,7 +2066,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), "python/loaders/orders.py": ( "from python.tasks.orders import prepare_orders\n" @@ -2073,7 +2078,7 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo " depends_on=(prepare_orders,),\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1}]\n" ), }, ) @@ -2124,21 +2129,21 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/orders.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), "python/loaders/orders.py": ( "from python.tasks.orders import prepare_orders\n" @@ -2150,7 +2155,7 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou " depends_on=(prepare_orders,),\n" ")\n" "def stage_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader stage_orders.'''\n return [{'order_id': 1}]\n" ), }, ) @@ -2204,13 +2209,13 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "from sqlbuild.checks import check\n\n" "@check(depends_on=prepare_orders)\n" "def check_orders_export(ctx):\n" - " return ctx.pass_()\n" + " '''Test check check_orders_export.'''\n return ctx.pass_()\n" ), "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result()\n" + " '''Test task prepare_orders.'''\n return ctx.result()\n" ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py index ff7086fcfd..eb609edd8d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/helpers.py @@ -22,7 +22,7 @@ ) _EVENTS_SEED_YML: str = ( "seeds:\n" - "- name: events\n" + "- name: events\n description: Test seed events.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -31,12 +31,12 @@ ) _EVENTS_CSV: str = "id,event_date\n1,2026-09-01\n2,2026-09-02\n3,2026-09-03\n" _UPSTREAM_SQL: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n id,\n CAST(event_date AS DATE) AS event_date\n" 'FROM __seed("events")\n' ) _DOWNSTREAM_MB_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor event_date,\n" @@ -59,7 +59,7 @@ 'SELECT\n id,\n event_date\nFROM __ref("upstream_v")\n' ) _DOWNSTREAM_MERGE_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -80,7 +80,7 @@ def _upstream_view_sql(*, upstream: str) -> str: return ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model udf__adjust_amount.',\n materialized view,\n);\n\n" f'SELECT\n id,\n event_date\nFROM __ref("{upstream}")\n' ) @@ -125,7 +125,7 @@ def insert_table_between_upstream_and_view(*, project_dir: Path) -> None: """Add a new table between the upstream table and its view.""" (project_dir / "models/upstream_new.sql").write_text( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model upstream_new.',\n materialized table,\n);\n\n" 'SELECT\n id,\n event_date\nFROM __ref("upstream")\n', encoding="utf-8", ) @@ -171,7 +171,7 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: "sqlbuild_project.toml": _PROJECT_TOML, "seeds/seeds.yml": ( "seeds:\n" - "- name: orders\n" + "- name: orders\n description: Test seed orders.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -184,10 +184,11 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: "id,ordered_at,amount\n1,2026-09-01,10\n2,2026-09-02,20\n3,2026-09-03,30\n" ), _ADJUST_FUNCTION_PATH: ( - "FUNCTION (\n arguments (amount INTEGER),\n returns INTEGER,\n);\n\namount + 1\n" + "FUNCTION (description 'Test function.'," + "\n arguments (amount INTEGER),\n returns INTEGER,\n);\n\namount + 1\n" ), "models/order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_amounts.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -202,7 +203,7 @@ def build_function_project(*, tmp_path: Path, caller_config: str) -> Path: 'FROM __seed("orders")\n' ), "models/order_totals.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_totals.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -233,7 +234,7 @@ def direct_reference_merge_files() -> dict[str, str]: return { "models/down_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -270,11 +271,11 @@ def build_star_project(*, tmp_path: Path, downstream_config: str) -> Path: repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/up.sql": ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model up.',\n materialized table,\n);\n\n" "SELECT 1 AS id, DATE '2026-09-01' AS event_date\n" ), "models/down.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down.',\n" " materialized incremental,\n" " incremental_strategy append,\n" " replay_on_change full,\n" @@ -332,10 +333,11 @@ def udf_reference_merge_files() -> dict[str, str]: return { "functions/sql/udf__plus_one.sql": ( - "FUNCTION (\n arguments (value INTEGER),\n returns INTEGER,\n);\n\nvalue + 1\n" + "FUNCTION (description 'Test function udf__plus_one.'," + "\n arguments (value INTEGER),\n returns INTEGER,\n);\n\nvalue + 1\n" ), "models/down_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model down_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -359,7 +361,7 @@ def incremental_upstream_files() -> dict[str, str]: return { "models/upstream.sql": ( - "MODEL (\n" + "MODEL (description 'Test model upstream.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [id],\n" @@ -397,7 +399,8 @@ def drop_upstream_column(*, project_dir: Path) -> None: """Drop the event_date column from the star project's upstream table.""" (project_dir / "models/up.sql").write_text( - "MODEL (\n materialized table,\n);\n\nSELECT 1 AS id\n", encoding="utf-8" + "MODEL (description 'Test model up.',\n materialized table,\n);\n\nSELECT 1 AS id\n", + encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py index eca6f20f34..a41b2e1e6d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py @@ -53,6 +53,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -60,7 +61,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, cursor ordered_at, @@ -180,6 +181,7 @@ def test_given_append_cursor_project_when_rerunning_build_then_boundary_behavior """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -187,7 +189,7 @@ def test_given_append_cursor_project_when_rerunning_build_then_boundary_behavior + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, cursor ordered_at, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py index e3de912241..259e54c649 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py @@ -41,7 +41,7 @@ def test_given_failing_factory_audit_when_building_then_existing_table_is_untouc materialized = "table" """, "models/orders.sql": """ -MODEL (audit_factories [order_quality]); +MODEL (description "Test model orders.", audit_factories [order_quality]); SELECT 2 AS id, -10 AS amount """, "audits/generic/expression_is_true.sql": """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py index ae4e460f6e..6a8674fac6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_failures.py @@ -61,7 +61,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy merge, unique_key [id], @@ -135,7 +135,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy merge, unique_key [id], @@ -210,7 +210,7 @@ def test_given_audit_failure_projects_when_running_build_then_cli_reports_failur '[connection]\ndatabase = "measurement.duckdb"\n' ), "models/orders.sql": ( - "MODEL (materialized table, audits [row_rate (" + "MODEL (description 'Test model orders.', materialized table, audits [row_rate (" "thresholds (error (below 90)))]) ; SELECT 1 AS order_id" ), "audits/generic/row_rate.sql": ( @@ -229,8 +229,12 @@ def test_given_audit_failure_projects_when_running_build_then_cli_reports_failur 'name = "measurement_audit"\nadapter = "duckdb"\n' '[connection]\ndatabase = "measurement.duckdb"\n' ), - "models/orders.sql": ("MODEL (materialized table); SELECT 1 AS order_id"), - "models/customers.sql": ("MODEL (materialized table); SELECT 1 AS order_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table); SELECT 1 AS order_id" + ), + "models/customers.sql": ( + "MODEL (description 'Test model customers.', materialized table); SELECT 1 AS order_id" + ), "audits/singular/rate.sql": ( "AUDIT (evaluation measurement, value rate, " "thresholds (error (below 90))); " diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 9ca3a1bad5..0e80b749db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -44,13 +44,13 @@ description="selected downstream clones missing upstream boundary from prod", project_name="defer_clone_build", initial_upstream_sql=( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'prod_version' AS label\n" + "MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id, 'prod_version' AS label\n" ), changed_upstream_sql=( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'dev_version' AS label\n" + "MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id, 'dev_version' AS label\n" ), downstream_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT id, label || '_downstream' AS label FROM __ref(\"upstream\")\n" ), prod_build_command=("--no-color", "build", "--target", "prod"), @@ -132,10 +132,10 @@ def test_given_changed_unselected_upstream_when_building_leaf_then_warns_and_noo "[connection]\n" f'database = "{test_case.project_name}.duckdb"\n' ), - "models/a.sql": "MODEL (materialized table);\n\nselect 1 as id\n", - "models/b.sql": "MODEL (materialized table);\n\nselect 1 as id\n", + "models/a.sql": "MODEL (description 'Test model a.', materialized table);\n\nselect 1 as id\n", + "models/b.sql": "MODEL (description 'Test model b.', materialized table);\n\nselect 1 as id\n", "models/c.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model c.', materialized table);\n\n" 'select * from __ref("a") union all select * from __ref("b")\n' ), }, @@ -146,11 +146,11 @@ def test_given_changed_unselected_upstream_when_building_leaf_then_warns_and_noo project_dir=project_dir, ) (project_dir / "models" / "a.sql").write_text( - "MODEL (materialized table);\n\nselect 10 as id\n", + "MODEL (description 'Test model a.', materialized table);\n\nselect 10 as id\n", encoding="utf-8", ) (project_dir / "models" / "b.sql").write_text( - "MODEL (materialized table);\n\nselect 2 as id\n", + "MODEL (description 'Test model b.', materialized table);\n\nselect 2 as id\n", encoding="utf-8", ) mixed_result: subprocess.CompletedProcess[str] = run_sqb( @@ -208,10 +208,10 @@ def test_given_direct_function_dependency_when_building_then_persists_function_h 'database = "warehouse.duckdb"\n' ), "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 100\n" + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 100\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT __udf("is_large_order")(150) AS is_large\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT __udf("is_large_order")(150) AS is_large\n' ), }, ) @@ -269,7 +269,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -277,7 +277,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist " query: SELECT data_version FROM freshness_control\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -298,7 +298,7 @@ def test_given_source_freshness_changes_during_build_when_appending_then_persist ) (project_dir / "models" / "orders.sql").write_text( ( - "MODEL (materialized table, " + "MODEL (description 'Test model orders.', materialized table, " 'post_hooks [inline_sql("UPDATE freshness_control SET data_version = 2")]);\n\n' 'SELECT * FROM __source("raw_orders")\n' ), @@ -359,13 +359,13 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 1 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" " freshness:\n" " strategy: sql\n" @@ -373,10 +373,10 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ " query: SELECT 1 AS data_version\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/payments.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_payments")\n' + 'MODEL (description "Test model payments.", materialized table);\n\nSELECT * FROM __source("raw_payments")\n' ), }, ) @@ -396,13 +396,13 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ ) (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 2 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" " freshness:\n" " strategy: sql\n" @@ -411,7 +411,7 @@ def test_given_independent_source_branch_failure_when_building_then_appends_succ encoding="utf-8", ) (project_dir / "models" / "payments.sql").write_text( - "MODEL (materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS payment_id\n", + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS payment_id\n", encoding="utf-8", ) execution_json_path: Path = project_dir / "target" / "failed-build.json" @@ -486,13 +486,13 @@ def test_given_source_freshness_error_when_building_then_blocks_only_affected_br customer_freshness_query="SELECT CURRENT_TIMESTAMP AS data_version", ), "models/stg_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.", materialized view);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), "models/dim_customers.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_customers")\n' + 'MODEL (description "Test model dim_customers.", materialized table);\n\nSELECT * FROM __source("raw_customers")\n' ), }, ) @@ -566,13 +566,13 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 1 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id, 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -580,7 +580,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " query: SELECT 1 AS data_version\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT o.order_id, p.payment_id FROM __source("raw_orders") o ' 'JOIN __source("raw_payments") p USING (order_id)\n' ), @@ -602,13 +602,13 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ) (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " freshness:\n" " strategy: sql\n" " type: integer\n" " query: SELECT 2 AS data_version\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id, 1 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -617,7 +617,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap encoding="utf-8", ) (project_dir / "models" / "fact_orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS order_id\n", + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT CAST('bad' AS INTEGER) AS order_id\n", encoding="utf-8", ) @@ -660,18 +660,18 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " raise RuntimeError('prepare failed')\n" + " '''Test task prepare_orders.'''\n raise RuntimeError('prepare failed')\n" ), "python/loaders/raw.py": ( "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -679,7 +679,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, expected_exit_code=1, @@ -707,18 +707,18 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" + " '''Test task prepare_orders.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), "python/loaders/raw.py": ( "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 1}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -726,7 +726,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), }, expected_exit_code=0, @@ -753,7 +753,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_result(ctx):\n" - " return ctx.result(payload={'bad': {'a', 'b'}})\n" + " '''Test task produce_bad_result.'''\n return ctx.result(payload={'bad': {'a', 'b'}})\n" ), }, expected_exit_code=1, @@ -777,7 +777,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_metadata(ctx):\n" - " return ctx.result(payload={}, metadata={'bad': {'a', 'b'}})\n" + " '''Test task produce_bad_metadata.'''\n return ctx.result(payload={}, metadata={'bad': {'a', 'b'}})\n" ), }, expected_exit_code=1, @@ -799,21 +799,21 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def fail_after_fact(ctx):\n" - " raise RuntimeError('profile failed')\n" + " '''Test task fail_after_fact.'''\n raise RuntimeError('profile failed')\n" ), }, expected_exit_code=1, @@ -838,14 +838,14 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" @@ -853,7 +853,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def skip_after_fact(ctx):\n" - " return ctx.skip(reason='no profile needed', mode=SkipMode.HARD)\n" + " '''Test task skip_after_fact.'''\n return ctx.skip(reason='no profile needed', mode=SkipMode.HARD)\n" ), }, expected_exit_code=0, @@ -878,14 +878,14 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" @@ -893,7 +893,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def soft_skip_after_fact(ctx):\n" - " return ctx.skip(reason='optional profile skipped', mode=SkipMode.SOFT)\n" + " '''Test task soft_skip_after_fact.'''\n return ctx.skip(reason='optional profile skipped', mode=SkipMode.SOFT)\n" ), }, expected_exit_code=0, @@ -921,7 +921,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test task prepare_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result(payload={'order_id': 3})\n" ), @@ -930,11 +930,11 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " return [{'order_id': 3}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 3}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -942,7 +942,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from pathlib import Path\n" @@ -950,7 +950,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('profile.txt')\n" + " '''Test task profile_fact.'''\n marker = Path(__file__).parents[2].joinpath('profile.txt')\n" " marker.write_text('profiled')\n" " return ctx.result()\n" ), @@ -979,11 +979,11 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 5}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 5}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -996,7 +996,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" - " relation = ctx.relation(source('raw_orders'))\n" + " '''Test task profile_raw_orders.'''\n relation = ctx.relation(source('raw_orders'))\n" " value = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" " marker = Path(__file__).parents[2].joinpath('source_profile.txt')\n" " marker.write_text(str(value))\n" @@ -1069,14 +1069,14 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, expected_exit_code=1, expected_output_fragments=("requires intermediate loader 'fetch_events'",), @@ -1097,7 +1097,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" + " '''Test task prepare_events.'''\n marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result()\n" ), @@ -1106,11 +1106,11 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "from python.tasks.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1175,14 +1175,14 @@ def test_given_direct_source_only_build_when_loader_has_ingress_then_enforces_so " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, expected_exit_code=0, expected_output_fragments=("source raw_events",), @@ -1240,7 +1240,7 @@ def test_given_existing_intermediate_target_when_building_source_only_then_reuse "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" - " return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" + " '''Test asset observed_orders.'''\n return ctx.result(metadata={'uri': 's3://orders'}, materialized=False)\n" ), }, expected_exit_code=0, @@ -1348,14 +1348,14 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" ")\n" "def load_window_orders(ctx):\n" - " return [{'order_id': 7}]\n" + " '''Test loader load_window_orders.'''\n return [{'order_id': 7}]\n" ), "python/tasks/prepare.py": ( "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" - " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" + " '''Test task prepare_orders.'''\n rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), "python/assets/prepare.py": ( @@ -1364,7 +1364,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" - " payload = ctx.result_of(node_function=prepare_orders).payload\n" + " '''Test asset publish_prepared_orders.'''\n payload = ctx.result_of(node_function=prepare_orders).payload\n" " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" @@ -1375,12 +1375,12 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" + " '''Test loader raw_orders.'''\n marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1388,14 +1388,14 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " type: INTEGER\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" + " '''Test task profile_fact_orders.'''\n relation = ctx.relation(model('fact_orders'))\n" " order_id = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), @@ -1404,7 +1404,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" + " '''Test asset export_fact_orders.'''\n payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), "python/tasks/notify.py": ( @@ -1413,7 +1413,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" - " payload = ctx.result_of(node_function=export_fact_orders).payload\n" + " '''Test task notify_fact_orders.'''\n payload = ctx.result_of(node_function=export_fact_orders).payload\n" " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" @@ -1500,7 +1500,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'first'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'first'})\n" ), }, ) @@ -1519,7 +1519,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 84}, metadata={'source': 'second'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 84}, metadata={'source': 'second'})\n" ), encoding="utf-8", ) @@ -1532,7 +1532,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " raise RuntimeError('producer failed after previous success')\n" + " '''Test task produce_result.'''\n raise RuntimeError('producer failed after previous success')\n" ), encoding="utf-8", ) @@ -1555,7 +1555,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" - " latest = ctx.result_of(node_function=produce_result)\n" + " '''Test task consume_result.'''\n latest = ctx.result_of(node_function=produce_result)\n" f" first = ctx.result_of(node_function=produce_result, run_id='{first_run_id}')\n" f" failed = ctx.result_of(node_function=produce_result, run_id='{failed_run_id}')\n" " history = ctx.results_of(node_function=produce_result, limit=2)\n" @@ -1643,11 +1643,11 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1655,7 +1655,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), "python/tasks/consume.py": ( "from pathlib import Path\n" @@ -1663,7 +1663,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_loader_result(ctx):\n" - " result = ctx.result_of(node_function=raw_events)\n" + " '''Test task consume_loader_result.'''\n result = ctx.result_of(node_function=raw_events)\n" " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" " output.write_text(\n" " f\"{result.metadata['loader_name']}:{result.metadata['source_name']}:\"\n" @@ -1712,11 +1712,11 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " raise RuntimeError('loader failed')\n" + " '''Test loader raw_events.'''\n raise RuntimeError('loader failed')\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1724,7 +1724,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), }, command=("--no-color", "build", "--select", "fact_events"), @@ -1746,18 +1746,18 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" + " '''Test task prepare_events.'''\n return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), "python/loaders/events.py": ( "from python.tasks.prepare import prepare_events\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_events,))\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1765,7 +1765,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader " type: INTEGER\n" ), "models/fact_events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' + 'MODEL (description "Test model fact_events.", materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), }, command=("--no-color", "build", "--select", "+fact_events"), @@ -1845,7 +1845,7 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'target': ctx.target})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'target': ctx.target})\n" ), "python/tasks/consume.py": ( "from pathlib import Path\n" @@ -1853,7 +1853,7 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" - " result = ctx.result_of(node_function=produce_result)\n" + " '''Test task consume_result.'''\n result = ctx.result_of(node_function=produce_result)\n" " output = Path(__file__).parents[2].joinpath('target_result.txt')\n" " output.write_text(str(result.payload['target']))\n" " return ctx.result()\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py index 766986e419..d606fca874 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py @@ -55,11 +55,11 @@ def test_given_monthly_producer_with_midmonth_max_when_building_capped_daily_con ).strip() + "\n", "sources/raw.yml": ( - "sources:\n - name: raw_meetings\n schema: main\n table: raw_meetings\n" + "sources:\n - name: raw_meetings\n description: Test source raw_meetings.\n schema: main\n table: raw_meetings\n" ), "models/monthly_meetings.sql": dedent( """ - MODEL ( + MODEL (description "Test model monthly_meetings.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -80,7 +80,7 @@ def test_given_monthly_producer_with_midmonth_max_when_building_capped_daily_con + "\n", "models/daily_suffix.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_suffix.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -380,9 +380,11 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ """ sources: - name: raw_late + description: Test source raw_late. schema: main table: raw_late - name: raw_early + description: Test source raw_early. schema: main table: raw_early """ @@ -390,7 +392,7 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ + "\n", "models/capped_late.sql": dedent( """ - MODEL ( + MODEL (description "Test model capped_late.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -417,14 +419,14 @@ def test_given_any_watermarks_and_consumer_end_when_capped_winner_is_later_then_ + "\n", "models/uncapped_early.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model uncapped_early.", materialized table); SELECT id, event_time FROM __source("raw_early") """ ).strip() + "\n", "models/any_consumer.sql": dedent( """ - MODEL ( + MODEL (description "Test model any_consumer.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 2cfca30829..a55b5a8880 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -148,6 +148,7 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "relative_path": "hooks/sql/select_value.sql", "definition_sql": "SELECT @value", "kwargs": {"value": 1}, + "description": "Test hook select_value.", }, {"type": "python", "name": "notify", "kwargs": {"message": "starting"}}, ), @@ -176,11 +177,11 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m "from sqlbuild.hooks import hook\n\n" "@hook\n" "def notify(ctx, message):\n" - " ctx.log(message)\n" + " '''Test hook notify.'''\n ctx.log(message)\n" ), - "hooks/sql/select_value.sql": "HOOK ();\n\nSELECT @value\n", + "hooks/sql/select_value.sql": "HOOK (description 'Test hook select_value.');\n\nSELECT @value\n", "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" ' pre_hooks [sql("select_value", value: 1), ' 'python("notify", message: "starting")],\n' @@ -216,7 +217,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m "type": "sql", "name": "select_value", "relative_path": "hooks/sql/select_value.sql", - "description": None, + "description": "Test hook select_value.", "sql": "SELECT @value", } assert hooks_by_name["notify"]["relative_path"] == "hooks/python/notify.py" @@ -231,7 +232,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m description="unknown materialization stops compile", repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (materialized tablex);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized tablex);\n\nSELECT 1 AS order_id\n", }, expected_code="P001", expected_message_fragment="unknown materialization 'tablex'", @@ -242,7 +243,7 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + "MODEL (description 'Test model orders.',\n materialized incremental,\n incremental_strategy append,\n" " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py index 9510a9d06b..be98fffbb3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py @@ -51,7 +51,7 @@ def test_given_project_capability_disabled_when_batch_concurrency_exceeds_one_th + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -73,7 +73,7 @@ def test_given_project_capability_disabled_when_batch_concurrency_exceeds_one_th ).strip() + "\n", "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n schema: main\n table: raw_events\n" ), }, ) @@ -120,6 +120,7 @@ def test_given_opt_in_concurrent_microbatch_when_building_twice_then_records_all """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -127,7 +128,7 @@ def test_given_opt_in_concurrent_microbatch_when_building_twice_then_records_all + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -245,6 +246,7 @@ def test_given_later_batches_succeed_when_one_batch_fails_then_next_run_preserve """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -252,7 +254,7 @@ def test_given_later_batches_succeed_when_one_batch_fails_then_next_run_preserve + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -402,6 +404,7 @@ def test_given_microbatch_history_is_lost_when_reconciling_then_policy_is_applie """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -409,7 +412,7 @@ def test_given_microbatch_history_is_lost_when_reconciling_then_policy_is_applie + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -511,6 +514,7 @@ def test_given_bounded_replay_on_change_when_model_changes_then_requirement_is_d """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -610,6 +614,7 @@ def test_given_full_replay_fails_when_retried_then_original_requirement_is_reuse """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -700,6 +705,7 @@ def test_given_version_hash_returns_when_replaying_then_new_transition_gets_new_ """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -790,6 +796,7 @@ def test_given_historical_backfill_when_normal_run_resumes_then_gap_to_normal_fl """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -797,7 +804,7 @@ def test_given_historical_backfill_when_normal_run_resumes_then_gap_to_normal_fl + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -898,6 +905,7 @@ def test_given_no_new_cursor_work_when_history_is_lost_then_reconciliation_still """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -905,7 +913,7 @@ def test_given_no_new_cursor_work_when_history_is_lost_then_reconciliation_still + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py index 9ac5975d75..559fae9c2b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py @@ -36,11 +36,11 @@ "(customer_id INTEGER, last_ordered_at TIMESTAMP);\n" ), "models/stg_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_orders.', materialized view);\n\n" "SELECT id AS order_id, ordered_at FROM main.raw_orders\n" ), "models/fact_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model fact_orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (\n" @@ -52,7 +52,7 @@ ), "models/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -94,7 +94,7 @@ ), "models/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [customer_id], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py index fefccc6727..aa73f3aca0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py @@ -48,7 +48,7 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t '"SQBRCONTRACT106"]\n' ), "models/stg_order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_amounts.',\n" " materialized table,\n" " columns (\n" " customer_id (type INTEGER),\n" @@ -67,7 +67,7 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t "CAST(7.00 AS DECIMAL(12,2))\n" ), "models/customer_category_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " materialized table,\n" " columns (customer_id (type INTEGER)),\n" " dynamic_columns (\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py index edc5ff6bb1..3a3bd5c27d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_enum_contract.py @@ -47,7 +47,7 @@ def test_given_enum_typed_contract_when_value_is_invalid_then_build_fails_audit( "models/_enums/fulfillment_method.sql": "ENUM (name fulfillment_method, members [DELIVERY, PICKUP]);\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized table, contract enforced, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py index 1889731764..9af51c2bcc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_execution_limits.py @@ -51,8 +51,8 @@ def test_given_target_model_limit_when_building_then_effective_policy_controls_e max_models = 1 remediation = "Automated tools must request approval before changing this policy." """.strip(), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "models/order_items.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id\n", + "models/order_items.sql": "MODEL (description 'Test model order_items.', materialized table);\n\nSELECT 1 AS id\n", }, ) @@ -112,8 +112,8 @@ def test_given_local_model_limit_override_when_building_then_local_policy_is_app [targets.dev.execution_limits] max_models = 2 """.strip(), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "models/order_items.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id\n", + "models/order_items.sql": "MODEL (description 'Test model order_items.', materialized table);\n\nSELECT 1 AS id\n", }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py index a8fd48689f..320f93323e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py @@ -40,7 +40,7 @@ EXPLICIT_REFERENCE_MACROS + "\n\ndef orders_base():\n return 'SELECT * FROM __ref(\"stg_orders_eu\")'\n" ) -_EMITTING_MODEL: str = "MODEL (materialized table);\n@orders_base()\n" +_EMITTING_MODEL: str = "MODEL (description 'Test model.', materialized table);\n@orders_base()\n" _SECOND_EMITTING_MACROS: str = ( "def customers_base():\n return 'SELECT * FROM __ref(\"stg_customers\")'\n" ) @@ -50,8 +50,10 @@ "AUDIT ();\nSELECT s.order_id FROM @relation s\n" 'LEFT JOIN __ref("{read}") a USING (order_id) WHERE a.order_id IS NULL\n' ) -_AUDITED_MODEL: str = "MODEL (materialized table, audits [{audit}]);\nSELECT 1 AS order_id\n" -_SUMMARY_MODEL: str = 'MODEL (materialized table);\nSELECT * FROM __ref("{read}")\n' +_AUDITED_MODEL: str = "MODEL (description 'Test model.', materialized table, audits [{audit}]);\nSELECT 1 AS order_id\n" +_SUMMARY_MODEL: str = ( + 'MODEL (description "Test model.", materialized table);\nSELECT * FROM __ref("{read}")\n' +) _ONE_MODEL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("{read}") WHERE order_id IS NULL\n' _NULL_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL\n' _LITERAL_TASK: str = ( @@ -59,14 +61,14 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" - ' ctx.query("SELECT count(*) FROM stg_customers")\n' + ' """Test task export_orders."""\n ctx.query("SELECT count(*) FROM stg_customers")\n' ) _RUNTIME_TASK: str = ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_orders(ctx):\n" - ' orders = ctx.relation(model("all_orders"))\n' + ' """Test task export_orders."""\n orders = ctx.relation(model("all_orders"))\n' ' ctx.query(f"SELECT count(*) FROM {orders}")\n' ' customers = "stg_" + "customers"\n' ' ctx.query(f"SELECT count(*) FROM {customers}")\n' @@ -75,7 +77,7 @@ _COUNTRY_SEED_FILES: dict[str, str] = { "seeds/countries.yml": ( - "seeds:\n - name: countries\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: countries\n description: Test seed countries.\n columns:\n - name: code\n type: VARCHAR\n" ), "seeds/countries.csv": "code\nGB\nFR\n", } @@ -84,7 +86,7 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=seed("countries"))\n' "def count_countries(ctx):\n" - ' countries = ctx.relation(seed("countries"))\n' + ' """Test task count_countries."""\n countries = ctx.relation(seed("countries"))\n' " ctx.execute_sql(\n" ' f"CREATE OR REPLACE TABLE country_counts AS SELECT count(*) AS n FROM {countries}"\n' " )\n" @@ -94,7 +96,7 @@ "from sqlbuild.refs import model\n\n\n" '@check(depends_on=model("all_orders"))\n' "def orders_present(ctx):\n" - ' orders = ctx.relation(model("all_orders"))\n' + ' """Test check orders_present."""\n orders = ctx.relation(model("all_orders"))\n' ' count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]\n' ' return ctx.pass_() if count else ctx.fail(message="no orders")\n' ) @@ -109,9 +111,9 @@ ) _RAW_SOURCES: str = ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" ) @@ -124,10 +126,10 @@ "from sqlbuild.tasks import task\n\n\n" '@task(depends_on=model("all_orders"))\n' "def export_summary(ctx):\n" - ' ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n' + ' """Test task export_summary."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_summary_ran AS SELECT 1 AS n")\n\n\n' '@task(depends_on=model("stg_orders_eu"))\n' "def export_eu(ctx):\n" - ' ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n' + ' """Test task export_eu."""\n ctx.execute_sql("CREATE OR REPLACE TABLE export_eu_ran AS SELECT 1 AS n")\n' ) @@ -205,7 +207,7 @@ def test_given_python_seed_and_model_dependencies_when_building_then_graph_selec UnrenderedMacroArgumentE2ETestCase( description="unchosen dictionary entry is a dependency the build selects", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" '@pick({"eu": __ref("stg_orders_eu"), "us": __ref("stg_orders_us")}, key="eu")\n' ), expected_dag_edges=( @@ -385,7 +387,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an expected_exit_code=1, expected_output_fragments=( "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'", - "--> python/tasks/export.py:7", + "--> python/tasks/export.py:8", "[references]\n enforce_explicit = false", ), ), @@ -396,7 +398,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an expected_exit_code=1, expected_output_fragments=( "error[P008]: check:orders_present names model:stg_customers as 'stg_customers'", - "--> python/checks/orders.py:8", + "--> python/checks/orders.py:9", 'declare it with depends_on=model("stg_customers")', ), ), @@ -570,7 +572,7 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit "models/sales/_sqlbuild/_macros/customers.py": _SECOND_EMITTING_MACROS, f"{EXPLICIT_REFERENCE_MODELS}/hidden.sql": _EMITTING_MODEL, f"{EXPLICIT_REFERENCE_MODELS}/hidden_customers.sql": ( - "MODEL (materialized table);\n@customers_base()\n" + "MODEL (description 'Test model hidden_customers.', materialized table);\n@customers_base()\n" ), "python/tasks/export.py": _LITERAL_TASK, }, @@ -649,11 +651,11 @@ def test_given_check_runtime_hard_coded_relation_when_building_then_it_warns_wit "audits/generic/us_check.sql": _NULL_AUDIT, f"{_AUDITS}/audits/generic/unused_check.sql": _NULL_AUDIT, f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_eu.sql": ( - "MODEL (materialized table, audits [eu_check]);\n" + "MODEL (description 'Test model stg_orders_eu.', materialized table, audits [eu_check]);\n" "SELECT 1 AS order_id, 10 AS amount\n" ), f"{EXPLICIT_REFERENCE_MODELS}/stg_orders_us.sql": ( - "MODEL (materialized table, audits [us_check]);\n" + "MODEL (description 'Test model stg_orders_us.', materialized table, audits [us_check]);\n" "SELECT 2 AS order_id, 20 AS amount\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py index d49a3ff666..99addcdb95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py @@ -42,6 +42,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, '1', '2026-01-01 00:30:00', 100, 'web'), @@ -64,7 +65,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py index d0b28ade69..a21a982ccb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py @@ -57,13 +57,13 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " maximum = int(Path(__file__).parents[2].joinpath('maximum.txt').read_text())\n" + " '''Test loader raw_orders.'''\n maximum = int(Path(__file__).parents[2].joinpath('maximum.txt').read_text())\n" " return [{'id': value, 'amount': value * 100} " "for value in range(1, maximum + 1)]\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -74,7 +74,7 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ ), "models/raw_orders_incremental.sql": dedent( """ - MODEL ( + MODEL (description "Test model raw_orders_incremental.", materialized incremental, incremental_strategy delete_insert, cursor id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py index b1803c99ca..3e6c5c4d3f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py @@ -40,7 +40,7 @@ def test_given_direct_project_when_building_then_manifest_requires_compile_manif "[connection]\n" 'database = "warehouse.duckdb"\n' ), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", }, ) manifest_path: Path = project_dir / "target" / "manifest.json" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py index 8af804249c..0fce894f98 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_aware_cursor_date_input.py @@ -31,6 +31,7 @@ """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events columns: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py index 0d71df9d74..5bd9329e48 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py @@ -132,7 +132,7 @@ def test_given_integer_cursor_maximum_when_building_then_half_open_batches_inclu ), "sources/raw.yml": raw_events_source_yml(), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py index b0ae8fb8ad..57fe340b2d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_mixed_timestamp_grain_replay.py @@ -71,6 +71,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -78,7 +79,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, @@ -91,7 +92,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, quantity, ordered_at, line_total_cents FROM __ref("stg_orders") @@ -100,7 +101,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -127,7 +128,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -154,7 +155,7 @@ + "\n", "models/marts/hourly_activity_with_daily_context.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_activity_with_daily_context.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py index b93ea51daf..fc0f0bbf94 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py @@ -59,6 +59,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -66,7 +67,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view ); @@ -83,7 +84,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table ); @@ -100,7 +101,7 @@ + "\n", "models/intermediate/order_status_index.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_status_index.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -124,7 +125,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -154,7 +155,7 @@ + "\n", "models/marts/fact_orders_view.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders_view.", materialized view ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py index de9a9eee00..588c072fbe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py @@ -28,7 +28,7 @@ "from sqlbuild.hooks import hook\n\n\n" "@hook\n" "def fail_hook(ctx, message):\n" - " raise RuntimeError(message)\n" + " '''Test hook fail_hook.'''\n raise RuntimeError(message)\n" ) @@ -74,13 +74,14 @@ def test_given_python_pre_hook_returns_skip_when_building_then_model_and_downstr @hook def skip_model(ctx): + '''Test hook skip_model.''' return ctx.skip(reason="source disabled") """ ).strip() + "\n", "models/upstream_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model upstream_orders.", materialized table, pre_hooks [python("skip_model")] ); @@ -91,7 +92,7 @@ def skip_model(ctx): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT order_id FROM __ref("upstream_orders") """ @@ -158,13 +159,14 @@ def test_given_python_post_hook_skip_when_building_then_keeps_relation_and_skips @hook def skip_downstream(ctx): + '''Test hook skip_downstream.''' return ctx.skip(reason="publish disabled") """ ).strip() + "\n", "models/upstream_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model upstream_orders.", materialized table, post_hooks [python("skip_downstream")] ); @@ -175,7 +177,7 @@ def skip_downstream(ctx): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT order_id FROM __ref("upstream_orders") """ @@ -245,6 +247,7 @@ def test_given_project_with_python_hooks_when_building_then_hooks_execute( @hook def create_hook_data(ctx, value): + '''Test hook create_hook_data.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.hook_data AS " f"SELECT {value} AS id, 'created by hook' AS label" @@ -253,6 +256,7 @@ def create_hook_data(ctx, value): @hook def record_hook_completion(ctx, phase): + '''Test hook record_hook_completion.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.hook_log AS " f"SELECT '{ctx.model_name}' AS model_name, " @@ -272,7 +276,7 @@ def record_hook_completion(ctx, phase): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [sql("select_one", value: 1), python("create_hook_data", value: 42)], post_hooks [python("record_hook_completion", phase: "post"), inline_sql("SELECT 1")] @@ -426,9 +430,11 @@ def test_given_python_hooks_lifecycle_matrix_when_building_then_all_materializat """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -441,6 +447,7 @@ def test_given_python_hooks_lifecycle_matrix_when_building_then_all_materializat @hook def log_hook(ctx): + '''Test hook log_hook.''' ctx.execute_sql( f"INSERT INTO {ctx.destination.schema}.hook_log VALUES " f"('{ctx.model_name}', '{ctx.phase}')" @@ -464,7 +471,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -483,7 +490,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -502,7 +509,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/intermediate/order_status_index.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_status_index.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -526,7 +533,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -555,7 +562,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/snapshots/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -571,7 +578,7 @@ def materialize(ctx: MaterializationContext) -> MaterializationResult: + "\n", "models/custom/custom_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model custom_orders.", materialized copy_table, pre_hooks [python("log_hook")], post_hooks [python("log_hook")] @@ -652,13 +659,14 @@ def test_given_python_post_hook_failure_when_building_graph_then_downstream_is_b @hook def fail_hook(ctx, message): + '''Test hook fail_hook.''' raise RuntimeError(message) """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [python("fail_hook", message: "intentional post failure")] ); @@ -669,7 +677,7 @@ def fail_hook(ctx, message): + "\n", "models/downstream_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model downstream_orders.", materialized table); SELECT id FROM __ref("orders") """ @@ -700,7 +708,7 @@ def fail_hook(ctx, message): description="pre hook failure shows failing Python hook row", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model.", materialized table, pre_hooks [python("fail_hook", message: "intentional pre failure")] ); @@ -722,7 +730,7 @@ def fail_hook(ctx, message): description="pre hook failure shows failing SQL hook row", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model lifecycle.", materialized table, pre_hooks [inline_sql("SELECT * FROM missing_hook_table")] ); @@ -820,10 +828,10 @@ def test_given_long_python_hook_name_when_building_then_cli_truncates_label_at_c "@hook\n" "def publish_customer_metadata_to_external_catalog_after_successful_" "materialization(ctx):\n" - ' ctx.log("long hook ran")\n' + ' """Test hook publish_customer_metadata_to_external_catalog_after_successful_materialization."""\n ctx.log("long hook ran")\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" ' pre_hooks [python("publish_customer_metadata_to_external_catalog_after_' 'successful_materialization")]\n' @@ -882,6 +890,7 @@ def test_given_snapshot_with_python_hooks_when_building_then_hooks_execute( @hook def create_snapshot_source(ctx): + '''Test hook create_snapshot_source.''' ctx.execute_sql( "CREATE TABLE main.raw_customers AS " "SELECT 1 AS customer_id, 'basic' AS plan, " @@ -891,6 +900,7 @@ def create_snapshot_source(ctx): @hook def log_snapshot_hook(ctx): + '''Test hook log_snapshot_hook.''' ctx.execute_sql( f"CREATE TABLE {ctx.destination.schema}.snapshot_hook_log AS " f"SELECT '{ctx.model_name}' AS model_name, '{ctx.phase}' AS phase" @@ -900,7 +910,7 @@ def log_snapshot_hook(ctx): + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index a2cc997b96..8d595bfc27 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -77,11 +77,13 @@ def test_given_built_python_nodes_when_editing_decorator_inputs_then_identity_tr @loader(write_strategy="table", columns=STAGED_ORDER_COLUMNS) def staged_orders(ctx): + '''Test loader staged_orders.''' return [{{"order_id": 1}}] @loader(depends_on=[staged_orders]) def raw_orders(ctx): + '''Test loader raw_orders.''' staged = ctx.loader(staged_orders).destination rows = ctx.query(f"SELECT order_id FROM {{staged}}") return [{{"order_id": row[0]}} for row in rows] @@ -91,6 +93,7 @@ def raw_orders(ctx): """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -108,6 +111,7 @@ def raw_orders(ctx): @asset(retry=EXPORT_RETRY) def export_orders(ctx): + '''Test asset export_orders.''' return None """ ).lstrip(), @@ -197,6 +201,7 @@ def normalize_status(value): @task def orders(ctx): + '''Test task orders.''' return ctx.result( payload={"status": python.helpers.clean.normalize_status(" Shipped ")} ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 2dbec73617..c792b221af 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -24,7 +24,7 @@ ) _FAILING_ORDERS_SQL: str = ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" "SELECT order_id FROM __source(\"raw_orders\") WHERE error('orders failed') IS NULL\n" ) _EXISTING_ORDERS: str = "CREATE TABLE main.orders AS SELECT * FROM range(1, 6) AS t(order_id);" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py index 5db2a5626a..244dbfa4ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_source_deferral.py @@ -28,33 +28,33 @@ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 9}]\n" ), "models/sales/_sqlbuild/_hooks/python/count.py": ( "from sqlbuild.hooks import hook\nfrom sqlbuild.refs import source\n\n\n" '@hook(reads=source("raw_orders"))\n' "def count_raw(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test hook count_raw."""\n raw = ctx.relation(source("raw_orders"))\n' ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.hook_counts AS ' 'SELECT count(*) AS n FROM {raw}")\n' ), "models/sales/order_count.sql": ( - 'MODEL (materialized table, post_hooks [python("count_raw")]);\n' + 'MODEL (description "Test model order_count.", materialized table, post_hooks [python("count_raw")]);\n' 'SELECT count(*) AS n FROM __source("raw_orders")\n' ), "models/sales/order_marker.sql": ( - 'MODEL (materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n' + 'MODEL (description "Test model order_marker.", materialized table, post_hooks [python("count_raw")]);\nSELECT 1 AS n\n' ), "python/tasks/count.py": ( "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" '@task(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test task count_task."""\n raw = ctx.relation(source("raw_orders"))\n' ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.task_counts AS ' 'SELECT count(*) AS n FROM {raw}")\n' ), @@ -62,7 +62,7 @@ "from sqlbuild.checks import check\nfrom sqlbuild.refs import source\n\n\n" '@check(depends_on=source("raw_orders"))\n' "def two_raw_orders(ctx):\n" - ' raw = ctx.relation(source("raw_orders"))\n' + ' """Test check two_raw_orders."""\n raw = ctx.relation(source("raw_orders"))\n' ' n = ctx.query(f"SELECT count(*) FROM {raw}").fetchone()[0]\n' ' return ctx.pass_() if n == 2 else ctx.fail(message=f"read {n} rows")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 3a730bbdae..4fe9306891 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -58,6 +58,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -65,7 +66,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized view ); @@ -80,7 +81,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized table ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py index 1453ee5c17..5be91ae74d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_fingerprint_changes.py @@ -26,18 +26,18 @@ '[defaults]\nmaterialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount_cents INTEGER), returns BOOLEAN);\n\namount_cents > 100\n" + "FUNCTION (description 'Test function is_large_order.', arguments (amount_cents INTEGER), returns BOOLEAN);\n\namount_cents > 100\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "SELECT id AS order_id, amount_cents, 'web order' AS channel\n" 'FROM __source("raw_orders")\n' ), "models/large_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model large_orders.', materialized table);\n\n" "SELECT order_id, is_large_order(amount_cents) AS is_large\n" 'FROM __ref("stg_orders")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py index f803825f7b..a8136a594a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py @@ -56,6 +56,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -63,7 +64,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, ordered_at, amount_cents FROM __source("raw_orders") """ @@ -71,7 +72,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at, amount_cents AS line_total_cents FROM __ref("stg_orders") @@ -80,7 +81,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -106,7 +107,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -179,6 +180,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -186,7 +188,7 @@ + "\n", "functions/sql/is_high_value_order.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function is_high_value_order.", arguments (amount_cents INTEGER), returns BOOLEAN, ); @@ -197,7 +199,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model fact_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -262,6 +264,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -269,7 +272,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, ordered_at, amount_cents FROM __source("raw_orders") """ @@ -277,7 +280,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at, amount_cents AS line_total_cents FROM __ref("stg_orders") @@ -286,7 +289,7 @@ + "\n", "models/marts/hourly_order_activity.sql": dedent( """ - MODEL ( + MODEL (description "Test model hourly_order_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -313,7 +316,7 @@ + "\n", "models/marts/daily_activity_rollup.sql": dedent( """ - MODEL ( + MODEL (description "Test model daily_activity_rollup.", materialized incremental, incremental_strategy delete_insert, cursor activity_day, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py index f224bc1f82..741056fb97 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py @@ -41,14 +41,15 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ ).lstrip(), - "models/order_regions.sql": "MODEL (materialized table);\n\nSELECT 1 AS region_id\n", + "models/order_regions.sql": "MODEL (description 'Test model order_regions.', materialized table);\n\nSELECT 1 AS region_id\n", "models/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -66,7 +67,7 @@ _DAILY_ORDERS_CONFIG: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -99,7 +100,7 @@ _SOURCE_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, @@ -118,7 +119,7 @@ _MICROBATCH_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -145,7 +146,7 @@ _WINDOWED_ORDERS_SQL: str = dedent( """ - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, cursor order_date, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py index a9a15bb666..16e872ba43 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_reusable_model_schemas.py @@ -47,6 +47,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -54,7 +55,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization + "\n", "schemas/events/base.sql": dedent( """ - SCHEMA ( + SCHEMA (description "Test schema.", name event, columns ( event_id (type INTEGER, nullable false, audits [not_null]), @@ -66,7 +67,7 @@ def test_given_reusable_inherited_schemas_when_building_then_all_materialization + "\n", "schemas/events/sourced.sql": dedent( """ - SCHEMA ( + SCHEMA (description "Test schema.", name sourced_event, extends event, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py index 26ff549b22..3f23f0de51 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py @@ -40,7 +40,7 @@ "models/marts/daily/orders.sql": _MART_MODEL, "models/marts/weekly/orders_weekly.sql": _MART_MODEL, "models/marts/_sqlbuild/schemas/order_shape.sql": ( - "SCHEMA (name order_shape, columns (order_id (type INTEGER, nullable false)));\n" + "SCHEMA (description 'Test schema.', name order_shape, columns (order_id (type INTEGER, nullable false)));\n" ), "models/marts/_sqlbuild/audits/generic/order_check.sql": ( 'AUDIT ();\n\nSELECT order_id FROM __ref("@model") WHERE order_id IS NULL\n' @@ -52,18 +52,18 @@ "WHERE w.order_id IS NULL\n" ), "models/marts/_sqlbuild/hooks/sql/record_start.sql": ( - "HOOK ();\n\nINSERT INTO main.hook_log VALUES ('sql')\n" + "HOOK (description 'Test hook record_start.');\n\nINSERT INTO main.hook_log VALUES ('sql')\n" ), "models/marts/_sqlbuild/hooks/python/lifecycle.py": ( "from sqlbuild.hooks import hook\n\n\n" "@hook\n" "def record_finish(ctx):\n" - " ctx.execute_sql(\"INSERT INTO main.hook_log VALUES ('python')\")\n" + " '''Test hook record_finish.'''\n ctx.execute_sql(\"INSERT INTO main.hook_log VALUES ('python')\")\n" ), } _SEED_YAML: str = ( "seeds:\n" - " - name: order_codes\n" + " - name: order_codes\n description: Test seed order_codes.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -192,7 +192,7 @@ def test_given_seed_audit_when_building_then_it_runs_after_load_and_gates_depend "seeds/order_codes.yml": _SEED_YAML, "seeds/order_codes.csv": test_case.seed_csv, "models/coded_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, label FROM __seed("order_codes")\n' + 'MODEL (description "Test model coded_orders.", materialized table);\n\nSELECT order_id, label FROM __seed("order_codes")\n' ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py index 0ff6254fa2..871ed4c383 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py @@ -54,7 +54,7 @@ def test_given_seed_watermark_when_seed_changes_then_incremental_consumes_new_hi + "\n", "seeds/schema.yml": ( "seeds:\n" - " - name: order_events\n" + " - name: order_events\n description: Test seed order_events.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -64,7 +64,7 @@ def test_given_seed_watermark_when_seed_changes_then_incremental_consumes_new_hi "seeds/order_events.csv": test_case.initial_seed, "models/order_events_incremental.sql": dedent( """ - MODEL ( + MODEL (description "Test model order_events_incremental.", materialized incremental, incremental_strategy delete_insert, cursor id, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py index b1a0c16b71..cac6c592c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py @@ -103,6 +103,7 @@ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -110,7 +111,7 @@ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -124,7 +125,7 @@ + "\n", "models/current_customer_plans.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model current_customer_plans.", materialized table); SELECT customer_id, plan FROM __ref("customer_snapshot") @@ -289,6 +290,7 @@ def test_given_snapshot_selector_when_excluding_downstream_then_only_snapshot_bu """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -296,7 +298,7 @@ def test_given_snapshot_selector_when_excluding_downstream_then_only_snapshot_bu + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -389,15 +391,19 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va """ sources: - name: raw_current_customers + description: Test source raw_current_customers. schema: main table: raw_current_customers - name: raw_historical_customers + description: Test source raw_historical_customers. schema: main table: raw_historical_customers - name: raw_initial_customers + description: Test source raw_initial_customers. schema: main table: raw_initial_customers - name: raw_audit_customers + description: Test source raw_audit_customers. schema: main table: raw_audit_customers """ @@ -405,7 +411,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/current_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model current_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -420,7 +426,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/historical_out_of_order_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model historical_out_of_order_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -436,7 +442,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/initial_updated_at_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model initial_updated_at_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -451,7 +457,7 @@ def test_given_snapshot_hooks_when_building_then_hooks_execute_and_history_is_va + "\n", "models/audited_customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model audited_customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -672,6 +678,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -679,7 +686,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -745,6 +752,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -752,7 +760,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -824,6 +832,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -831,7 +840,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -918,6 +927,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -925,7 +935,7 @@ def test_given_advanced_snapshot_edges_when_building_then_history_remains_valid( + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1092,15 +1102,19 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( """ sources: - name: raw_current_timestamp_customers + description: Test source raw_current_timestamp_customers. schema: main table: raw_current_timestamp_customers - name: raw_current_check_customers + description: Test source raw_current_check_customers. schema: main table: raw_current_check_customers - name: raw_historical_snapshot_customers + description: Test source raw_historical_snapshot_customers. schema: main table: raw_historical_snapshot_customers - name: raw_historical_change_customers + description: Test source raw_historical_change_customers. schema: main table: raw_historical_change_customers """ @@ -1108,7 +1122,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_timestamp_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_timestamp_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1122,7 +1136,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_status_check_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_check_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -1136,7 +1150,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_historical_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_historical_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1152,7 +1166,7 @@ def test_given_snapshot_failure_when_building_then_history_remains_consistent( + "\n", "models/customer_plan_change_records_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_plan_change_records_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1560,15 +1574,19 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns """ sources: - name: raw_customer_regions + description: Test source raw_customer_regions. schema: main table: raw_customer_regions - name: raw_customer_deletes + description: Test source raw_customer_deletes. schema: main table: raw_customer_deletes - name: raw_customer_status_history + description: Test source raw_customer_status_history. schema: main table: raw_customer_status_history - name: raw_customer_status_hard_delete_history + description: Test source raw_customer_status_hard_delete_history. schema: main table: raw_customer_status_hard_delete_history """ @@ -1576,7 +1594,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_region_custom_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_region_custom_snapshot.", materialized snapshot, unique_key [customer_id, region], snapshot_strategy timestamp, @@ -1592,7 +1610,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1607,7 +1625,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_status_historical_check_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_historical_check_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -1622,7 +1640,7 @@ def test_given_shallow_waffle_shop_snapshots_when_sources_change_then_cli_reruns + "\n", "models/customer_status_historical_check_hard_delete_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_historical_check_hard_delete_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2000,9 +2018,11 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t """ sources: - name: raw_membership_observations + description: Test source raw_membership_observations. schema: main table: raw_membership_observations - name: raw_membership_full_history + description: Test source raw_membership_full_history. schema: main table: raw_membership_full_history """ @@ -2010,7 +2030,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_incremental_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_incremental_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2026,7 +2046,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_full_history_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_full_history_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -2111,9 +2131,11 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t """ sources: - name: raw_membership_observations + description: Test source raw_membership_observations. schema: main table: raw_membership_observations - name: raw_membership_full_history + description: Test source raw_membership_full_history. schema: main table: raw_membership_full_history """ @@ -2121,7 +2143,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_incremental_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_incremental_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2138,7 +2160,7 @@ def test_given_shallow_waffle_shop_snapshot_edges_when_sources_change_then_cli_t + "\n", "models/membership_full_history_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model membership_full_history_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2289,6 +2311,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2296,7 +2319,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2366,6 +2389,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2373,7 +2397,7 @@ def test_given_hard_deleted_key_when_it_reappears_unchanged_then_snapshot_opens_ + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2490,6 +2514,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2497,7 +2522,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2544,6 +2569,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2551,7 +2577,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2603,6 +2629,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2610,7 +2637,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2662,6 +2689,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2669,7 +2697,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2712,6 +2740,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2719,7 +2748,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2765,6 +2794,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2772,7 +2802,7 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2847,6 +2877,7 @@ def test_given_duplicate_timestamp_snapshot_source_when_building_then_cli_report """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2854,7 +2885,7 @@ def test_given_duplicate_timestamp_snapshot_source_when_building_then_cli_report + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -2965,6 +2996,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -2972,7 +3004,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3039,6 +3071,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3046,7 +3079,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3102,6 +3135,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3109,7 +3143,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3165,6 +3199,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3172,7 +3207,7 @@ def test_given_snapshot_dml_failure_when_building_then_target_history_is_unchang + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3288,6 +3323,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3295,7 +3331,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3366,6 +3402,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3373,7 +3410,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3443,6 +3480,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -3450,7 +3488,7 @@ def test_given_check_snapshot_project_when_rerunning_build_then_tracks_checked_c + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3568,6 +3606,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -3575,7 +3614,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3647,6 +3686,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -3654,7 +3694,7 @@ def test_given_historical_check_snapshot_project_when_rerunning_build_then_track + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3778,6 +3818,7 @@ def test_given_historical_timestamp_snapshot_project_when_rerunning_build_then_t """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -3785,7 +3826,7 @@ def test_given_historical_timestamp_snapshot_project_when_rerunning_build_then_t + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -3904,6 +3945,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3911,7 +3953,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -3954,6 +3996,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -3961,7 +4004,7 @@ def test_given_historical_timestamp_changes_project_when_rerunning_build_then_tr + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4036,6 +4079,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4043,7 +4087,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4088,6 +4132,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4095,7 +4140,7 @@ def test_given_missing_check_snapshot_output_column_when_building_then_cli_repor + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4184,6 +4229,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4191,7 +4237,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4249,6 +4295,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customers + description: Test source raw_customers. schema: main table: raw_customers """ @@ -4256,7 +4303,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4310,6 +4357,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4317,7 +4365,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4388,6 +4436,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4395,7 +4444,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4466,6 +4515,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4473,7 +4523,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4550,6 +4600,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4557,7 +4608,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4633,6 +4684,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -4640,7 +4692,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4717,6 +4769,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_changes + description: Test source raw_customer_changes. schema: main table: raw_customer_changes """ @@ -4724,7 +4777,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -4800,6 +4853,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_daily + description: Test source raw_customer_daily. schema: main table: raw_customer_daily """ @@ -4807,7 +4861,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -4884,6 +4938,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports """ sources: - name: raw_customer_extracts + description: Test source raw_customer_extracts. schema: main table: raw_customer_extracts """ @@ -4891,7 +4946,7 @@ def test_given_snapshot_full_refresh_default_deny_when_building_then_cli_reports + "\n", "models/customer_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py index 86f73a6061..380a026955 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py @@ -77,7 +77,7 @@ def test_given_built_model_with_sql_hook_when_editing_hook_then_plans_expected_r _HOOK_PATH: _HOOK_CONTENTS, "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [sql("record_order_access", label: 1)] ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py index 6e005aff84..4f9231024e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_reads.py @@ -30,15 +30,15 @@ '[targets.prod]\nschema = "prod"\nloader_schema = "raw_prod"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 9}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 9}]\n" ), f"{_MODELS}/z_orders.sql": ( - "MODEL (materialized table);\nSELECT 1 AS order_id UNION ALL SELECT 2\n" + "MODEL (description 'Test model z_orders.', materialized table);\nSELECT 1 AS order_id UNION ALL SELECT 2\n" ), f"{_MODELS}/_sqlbuild/_macros/counts.py": ( "def count_orders(relation):\n" @@ -46,16 +46,16 @@ 'SELECT count(*) AS n FROM {relation}"\n' ), f"{_MODELS}/_sqlbuild/_hooks/sql/record_counts.sql": ( - 'HOOK ();\n@count_orders(__ref("z_orders"))\n' + 'HOOK (description "Test hook record_counts.");\n@count_orders(__ref("z_orders"))\n' ), f"{_MODELS}/a_report.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model a_report.',\n materialized table,\n" " pre_hooks [inline_sql('CREATE OR REPLACE TABLE main.inline_counts AS " 'SELECT count(*) AS n FROM __ref("z_orders")\')],\n' ' post_hooks [sql("record_counts")],\n);\nSELECT 1 AS n\n' ), f"{_MODELS}/raw_marker.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model raw_marker.',\n materialized table,\n" " post_hooks [inline_sql('CREATE OR REPLACE TABLE main.raw_counts AS " 'SELECT count(*) AS n FROM __source("raw_orders")\')],\n);\nSELECT 1 AS n\n' ), @@ -126,10 +126,10 @@ def test_given_sql_hook_references_when_building_then_they_resolve_and_order_lik description="inline hook reading a model built from its own model", overrides={ f"{_MODELS}/report_rollup.sql": ( - 'MODEL (materialized table);\nSELECT * FROM __ref("a_report")\n' + 'MODEL (description "Test model report_rollup.", materialized table);\nSELECT * FROM __ref("a_report")\n' ), f"{_MODELS}/a_report.sql": ( - "MODEL (\n materialized table,\n" + "MODEL (description 'Test model a_report.',\n materialized table,\n" " post_hooks [inline_sql('SELECT * FROM __ref(\"report_rollup\")')],\n" ");\nSELECT 1 AS n\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py index ea592a4c7e..608578127b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py @@ -49,7 +49,7 @@ def test_given_model_table_function_dependency_when_building_then_creates_and_co + "\n", "functions/sql/customer_orders.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table ( customer_id INTEGER, @@ -63,7 +63,7 @@ def test_given_model_table_function_dependency_when_building_then_creates_and_co + "\n", "models/customer_order_summary.sql": dedent( f""" - MODEL ( + MODEL (description "Test model customer_order_summary.", materialized table ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py index f3245ea09d..f1d336126f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py @@ -66,6 +66,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -73,7 +74,7 @@ + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", cursor ordered_at, cursor_type timestamp, cursor_grain second, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py index 1e2db0e347..26f27dd4bf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py @@ -36,7 +36,7 @@ "seed_raw_data.sql": "", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [inline_sql('THIS IS NOT VALID SQL')] ); @@ -89,6 +89,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -96,7 +97,7 @@ + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT id AS order_id, @@ -107,7 +108,7 @@ + "\n", "models/marts/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, ordered_at FROM __ref("stg_orders") """ @@ -115,7 +116,7 @@ + "\n", "models/marts/customer_status_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_status_snapshot.", materialized incremental, incremental_strategy merge, unique_key [order_id], @@ -157,13 +158,14 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: SELECT FROM """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model orders.", materialized table); SELECT * FROM __source("raw_orders") """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py index ce59cfcd15..0471c3810d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py @@ -29,11 +29,13 @@ def prepare_python_check_project(*, tmp_path: Path) -> Path: @task() def export_orders(ctx): + '''Test task export_orders.''' return ctx.result(payload={"rows": 3}, metadata={"rows": 3}) @task() def export_customers(ctx): + '''Test task export_customers.''' return ctx.result(payload={"rows": 2}, metadata={"rows": 2}) """, "python/assets/orders.py": """ @@ -43,6 +45,7 @@ def export_customers(ctx): @asset(depends_on=export_orders) def orders_asset(ctx): + '''Test asset orders_asset.''' return ctx.result(payload={"asset_rows": 3}, metadata={"asset_rows": 3}) """, "python/checks/orders.py": """ @@ -53,21 +56,25 @@ def orders_asset(ctx): @check(depends_on=export_orders) def check_orders_export(ctx): + '''Test check check_orders_export.''' return ctx.pass_(message="orders exported") @check(depends_on=export_orders, severity="warn") def warn_orders_export(ctx): + '''Test check warn_orders_export.''' return ctx.fail(message="warning check failed") @check(depends_on=orders_asset, tags=("asset",), group="python-checks") def check_orders_asset(ctx): + '''Test check check_orders_asset.''' return ctx.result_of(node_function=orders_asset).payload["asset_rows"] == 3 @check(depends_on=(export_orders, export_customers), tags=("multi",), group="python-checks") def check_order_customer_exports(ctx): + '''Test check check_order_customer_exports.''' return ctx.pass_( metadata={"orders": ctx.result_of(node_function=export_orders).metadata["rows"]} ) @@ -75,16 +82,19 @@ def check_order_customer_exports(ctx): @check(depends_on=[export_orders], severity="error", tags=("failure",), group="python-checks") def fail_orders_export(ctx): + '''Test check fail_orders_export.''' return ctx.fail(message="orders export failed") @check(depends_on=export_orders, severity="error", tags=("failure",), group="python-checks") def false_orders_export(ctx): + '''Test check false_orders_export.''' return False @check(depends_on=export_orders, severity="error", tags=("failure",), group="python-checks") def exception_orders_export(ctx): + '''Test check exception_orders_export.''' raise RuntimeError("orders exception check failed") """, }, @@ -111,11 +121,13 @@ def prepare_terminal_loader_check_project(*, tmp_path: Path) -> Path: @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 1}] """, "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -129,6 +141,7 @@ def raw_orders(ctx): @check(depends_on=raw_orders) def check_raw_orders_loader(ctx): + '''Test check check_raw_orders_loader.''' return True """, }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 332c5d933d..7e3220b3fc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -60,6 +60,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, 100), (2, 200)) @@ -69,7 +70,7 @@ + "\n", "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT order_id, amount_cents FROM __source("raw_orders") """ @@ -77,7 +78,7 @@ + "\n", "models/orders_enriched.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model orders_enriched.", materialized view); SELECT order_id, amount_cents, amount_cents * 2 AS doubled_cents FROM __ref("fact_orders") @@ -86,7 +87,7 @@ + "\n", "models/missing_snapshot.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model missing_snapshot.", materialized table); SELECT 1 AS id """ @@ -202,7 +203,7 @@ def test_given_clone_command_when_running_then_managed_relations_sync_as_expecte ).strip() + "\n", "models/fact_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id, 100 AS amount_cents\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT 1 AS order_id, 100 AS amount_cents\n" ), }, clone_command=( @@ -304,6 +305,7 @@ def test_given_invalid_origin_connection_when_exact_cloning_then_destination_ses @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [] """ ).strip() @@ -312,6 +314,7 @@ def raw_customers(ctx): """ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -324,7 +327,7 @@ def raw_customers(ctx): + "\n", "models/stg_customers.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_customers.", materialized view); SELECT customer_id, first_name FROM __source("raw_customers") """ @@ -384,6 +387,7 @@ def raw_customers(ctx): @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [] """ ).strip() @@ -392,6 +396,7 @@ def raw_customers(ctx): """ sources: - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -404,7 +409,7 @@ def raw_customers(ctx): + "\n", "models/stg_customers.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_customers.", materialized view); SELECT customer_id, first_name FROM __source("raw_customers") """ @@ -545,6 +550,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que """ seeds: - name: bonuses + description: Test seed bonuses. schema: seed_origin columns: - name: bonus @@ -554,7 +560,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table, schema model_origin); + MODEL (description "Test model orders.", materialized table, schema model_origin); SELECT * FROM (VALUES (1, 10), (2, 20)) AS orders(order_id, amount) """ @@ -562,7 +568,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "functions/sql/add_bonus.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function add_bonus.", schema scalar_origin, arguments (amount INTEGER), returns INTEGER @@ -574,7 +580,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "functions/sql/order_rows.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function order_rows.", schema table_function_origin, arguments (minimum_id INTEGER), returns table ( @@ -591,7 +597,7 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que + "\n", "models/enriched_orders.sql": dedent( """ - MODEL (materialized view, schema view_origin); + MODEL (description "Test model enriched_orders.", materialized view, schema view_origin); SELECT order_id, __udf("add_bonus")(amount) AS amount_with_bonus FROM __table_fn("order_rows")(1) @@ -678,6 +684,7 @@ def test_given_python_udf_when_cloning_then_recreates_for_clone_connection_scope runtime_version="3.11", ) def main(value): + '''Test function main.''' return value + 1 """ ).strip() @@ -759,6 +766,7 @@ def test_given_destination_source_deferral_when_cloning_then_function_reads_defe @loader def raw_bonus(ctx): + '''Test loader raw_bonus.''' return [] """ ).strip() @@ -767,6 +775,7 @@ def raw_bonus(ctx): """ sources: - name: raw_bonus + description: Test source raw_bonus. managed: true write_strategy: table columns: @@ -777,7 +786,7 @@ def raw_bonus(ctx): + "\n", "functions/sql/add_deferred_bonus.sql": dedent( """ - FUNCTION (arguments (amount INTEGER), returns INTEGER); + FUNCTION (description "Test function add_deferred_bonus.", arguments (amount INTEGER), returns INTEGER); amount + (SELECT bonus FROM __source("raw_bonus")) """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index cbe17445f1..98a0dbe6ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -496,3 +496,46 @@ class UnionFixtureCompileScalingTestCase: expected_sql_tests: int expected_errors: int expected_max_scaling_ratio: float + + +@dataclass(frozen=True) +class RequiredDescriptionCase: + description: str + missing_files: tuple[tuple[str, str], ...] + described_files: tuple[tuple[str, str], ...] + expected_diagnostic: tuple[str, str, int] + expected_message: str + expected_help_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RequiredDescriptionAggregateCase: + description: str + files: tuple[tuple[str, str], ...] + expected_codes: frozenset[str] + expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class RequiredDescriptionPlanCase: + description: str + model_sql: str + expected_returncode: int + expected_stderr_fragment: str + + +@dataclass(frozen=True) +class PathDefaultDescriptionCase: + description: str + model_files: tuple[tuple[str, str], ...] + expected_returncode: int + expected_diagnostics: tuple[object, ...] + expected_descriptions: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class FormatDescriptionResolutionE2ECase: + description: str + extra_files: tuple[tuple[str, str], ...] + expected_returncode: int + expected_faults: tuple[tuple[str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json index 37c0499799..97bcc86c90 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json @@ -7,7 +7,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\")\n" } }, { @@ -17,7 +17,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q, q + 1 AS q2 FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS q, q + 1 AS q2 FROM __source(\"raw__orders\")\n" } }, { @@ -27,7 +27,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > '2026-01-01'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > '2026-01-01'\n" } }, { @@ -37,7 +37,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ALL\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ALL\n" } }, { @@ -47,7 +47,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (status) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * EXCLUDE (status) FROM __source(\"raw__orders\")\n" } }, { @@ -57,7 +57,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * 2.5 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * 2.5 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -67,7 +67,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id, p.amount_cents FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id, p.amount_cents FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" } }, { @@ -77,7 +77,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q FROM __source(\"raw__orders\") ORDER BY q\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS q FROM __source(\"raw__orders\") ORDER BY q\n" } }, { @@ -87,7 +87,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.customer_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.customer_id)\n" } }, { @@ -97,7 +97,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" } }, { @@ -107,7 +107,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(ordered_at) AS a FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT AVG(ordered_at) AS a FROM __source(\"raw__orders\")\n" } }, { @@ -117,7 +117,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity\n" } }, { @@ -127,7 +127,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NOT quantity AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NOT quantity AS x FROM __source(\"raw__orders\")\n" } }, { @@ -137,7 +137,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at AS x FROM __source(\"raw__orders\") UNION ALL SELECT email FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at AS x FROM __source(\"raw__orders\") UNION ALL SELECT email FROM __source(\"raw__customers\")\n" } }, { @@ -147,7 +147,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT TRUE FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT TRUE FROM __source(\"raw__customers\")\n" } }, { @@ -157,7 +157,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, id FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, id FROM __source(\"raw__orders\")\n" } }, { @@ -167,7 +167,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b) AS (SELECT id FROM __source(\"raw__orders\")) SELECT a FROM c\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH c(a, b) AS (SELECT id FROM __source(\"raw__orders\")) SELECT a FROM c\n" } }, { @@ -177,7 +177,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity + 1 FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity + 1 FROM __source(\"raw__orders\")\n" } }, { @@ -187,7 +187,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status = 'x') AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status = 'x') AS s FROM __source(\"raw__orders\")\n" } }, { @@ -197,7 +197,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity AND status = 'x'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity AND status = 'x'\n" } }, { @@ -207,7 +207,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id\n" } }, { @@ -217,7 +217,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b, d) AS (SELECT id, status FROM __source(\"raw__orders\")) SELECT a FROM c\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH c(a, b, d) AS (SELECT id, status FROM __source(\"raw__orders\")) SELECT a FROM c\n" } }, { @@ -227,7 +227,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" } }, { @@ -237,14 +237,14 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DISTINCT status FROM __source(\"raw__orders\") ORDER BY quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DISTINCT status FROM __source(\"raw__orders\") ORDER BY quantity\n" } }, { "description": "control: declared scalar UDF return type", "category": "control", "repo_files": { - "models/probe.sql": "MODEL (materialized table, contract enforced, columns (valid (type BOOLEAN)));\nSELECT __udf(\"udf__is_completed_order\")(status) AS valid FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table, contract enforced, columns (valid (type BOOLEAN)));\nSELECT __udf(\"udf__is_completed_order\")(status) AS valid FROM __source(\"raw__orders\")\n" }, "expected_exit_code": 0, "expected_codes": [], @@ -257,7 +257,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.id\n" } } ], @@ -271,7 +271,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __source(\"raw__orders\")\n" } }, { @@ -283,7 +283,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.qty FROM __source(\"raw__orders\") o\n" } }, { @@ -295,7 +295,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __ref(\"stg_orders\")\n" } }, { @@ -307,7 +307,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __ref(\"stg_orders_typed\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.qty FROM __ref(\"stg_orders_typed\") o\n" } }, { @@ -319,7 +319,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT amount_usd FROM __source(\"raw__payments\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT amount_usd FROM __source(\"raw__payments\")\n" } }, { @@ -331,7 +331,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT price_usd FROM __seed(\"waffle_types\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT price_usd FROM __seed(\"waffle_types\")\n" } }, { @@ -343,7 +343,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE qty > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE qty > 1\n" } }, { @@ -355,7 +355,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY stat\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY stat\n" } }, { @@ -367,7 +367,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY qty\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY qty\n" } }, { @@ -379,7 +379,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer = c.id\n" } }, { @@ -391,7 +391,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) AS rn FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) AS rn FROM __source(\"raw__orders\")\n" } }, { @@ -403,7 +403,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) = 1\n" } }, { @@ -415,7 +415,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status HAVING SUM(qty) > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status HAVING SUM(qty) > 1\n" } }, { @@ -427,7 +427,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN qty > 1 THEN 'a' END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN qty > 1 THEN 'a' END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -439,7 +439,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(stat) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(stat) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -451,7 +451,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT id FROM __source(\"raw__orders\")) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT id FROM __source(\"raw__orders\")) SELECT quantity FROM x\n" } }, { @@ -463,7 +463,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT s.quantity FROM (SELECT id FROM __source(\"raw__orders\")) s\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT s.quantity FROM (SELECT id FROM __source(\"raw__orders\")) s\n" } }, { @@ -475,7 +475,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.id FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z.id FROM __source(\"raw__orders\") o\n" } }, { @@ -487,7 +487,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.cust)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.cust)\n" } }, { @@ -499,7 +499,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT cust_id FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT cust_id FROM __source(\"raw__customers\"))\n" } }, { @@ -511,7 +511,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE p.amount > 0\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE p.amount > 0\n" } }, { @@ -523,7 +523,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c USING (cust_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c USING (cust_id)\n" } }, { @@ -535,7 +535,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + "models/marts/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" } }, { @@ -547,7 +547,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(stat) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(stat) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -559,7 +559,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" } }, { @@ -571,7 +571,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" } }, { @@ -583,7 +583,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON id = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON id = c.id\n" } }, { @@ -595,7 +595,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE order_id > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE order_id > 1\n" } }, { @@ -607,7 +607,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -619,7 +619,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, quantity, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, quantity, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status\n" } }, { @@ -631,7 +631,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE SUM(quantity) > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE SUM(quantity) > 1\n" } }, { @@ -643,7 +643,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(COUNT(*)) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(COUNT(*)) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -655,7 +655,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ROW_NUMBER() OVER () = 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ROW_NUMBER() OVER () = 1\n" } }, { @@ -667,7 +667,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, COUNT(*) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, COUNT(*) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -679,7 +679,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING quantity > 1\n" } }, { @@ -691,7 +691,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY 3\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY 3\n" } }, { @@ -703,7 +703,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status, id) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status, id) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -715,7 +715,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT 1 AS id) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT 1 AS id) SELECT quantity FROM x\n" } }, { @@ -727,7 +727,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE qty > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE qty > 1\n" } }, { @@ -739,7 +739,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle = w.waffle_type_id\n" } }, { @@ -751,7 +751,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT waffle_type_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT waffle_type_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" } }, { @@ -763,7 +763,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT order_id FROM __ref(\"stg_orders_typed\")) SELECT quantity FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT order_id FROM __ref(\"stg_orders_typed\")) SELECT quantity FROM x\n" } }, { @@ -775,7 +775,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT qty FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -787,7 +787,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT w.price_usd FROM __source(\"raw__orders\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT w.price_usd FROM __source(\"raw__orders\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" } }, { @@ -799,7 +799,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" } }, { @@ -823,7 +823,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type VARCHAR),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type VARCHAR),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -848,8 +848,8 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n qty (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity AS qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n", - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity FROM __ref(\"stg_orders_typed\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n qty (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity AS qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n", + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -885,7 +885,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY SUM(quantity)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY SUM(quantity)\n" } }, { @@ -897,7 +897,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE qty > 1) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) FILTER (WHERE qty > 1) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -909,7 +909,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON SUM(o.customer_id) = c.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON SUM(o.customer_id) = c.id\n" } }, { @@ -921,7 +921,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ROW_NUMBER() OVER ()) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(ROW_NUMBER() OVER ()) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -933,7 +933,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ROW_NUMBER() OVER ()\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ROW_NUMBER() OVER ()\n" } }, { @@ -945,7 +945,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity) OVER (ORDER BY ordered) AS l FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LAG(quantity) OVER (ORDER BY ordered) AS l FROM __source(\"raw__orders\")\n" } }, { @@ -955,7 +955,7 @@ "expected_codes": [], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(status, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(status, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -967,7 +967,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.id FROM (SELECT id FROM __source(\"raw__orders\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT x.id FROM (SELECT id FROM __source(\"raw__orders\"))\n" } }, { @@ -979,7 +979,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = z.customer_id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = z.customer_id)\n" } }, { @@ -991,7 +991,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id, l.x FROM __source(\"raw__orders\") o, LATERAL (SELECT o.qty AS x) l\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id, l.x FROM __source(\"raw__orders\") o, LATERAL (SELECT o.qty AS x) l\n" } }, { @@ -1003,7 +1003,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" } }, { @@ -1015,7 +1015,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z FROM (SELECT id FROM __source(\"raw__orders\") UNION ALL BY NAME SELECT id FROM __source(\"raw__customers\")) u\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z FROM (SELECT id FROM __source(\"raw__orders\") UNION ALL BY NAME SELECT id FROM __source(\"raw__customers\")) u\n" } }, { @@ -1027,7 +1027,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status ORDER BY quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status ORDER BY quantity\n" } }, { @@ -1039,7 +1039,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")() AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")() AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1051,7 +1051,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.amount FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.amount FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" } }, { @@ -1063,7 +1063,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")(1, 2) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")(1, 2) t\n" } }, { @@ -1075,7 +1075,7 @@ ], "pending_native": false, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(p.amount_usd)') AS d FROM __ref(\"stg_payments\") p\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT @cents_to_dollars('SUM(p.amount_usd)') AS d FROM __ref(\"stg_payments\") p\n" } }, { @@ -1267,7 +1267,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -1279,7 +1279,7 @@ ], "pending_native": false, "repo_files": { - "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type STRINGY),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + "models/staging/stg_orders_typed.sql": "MODEL (description \"Test model stg_orders_typed.\",\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type STRINGY),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" } }, { @@ -1394,7 +1394,7 @@ "description": "metadata-udf: scalar UDF declared argument type", "category": "udf", "repo_files": { - "models/probe.sql": "MODEL (materialized table);\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS valid FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS valid FROM __source(\"raw__orders\")\n" }, "expected_exit_code": 1, "expected_codes": [ @@ -1411,7 +1411,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.email\n" } }, { @@ -1423,7 +1423,7 @@ ], "pending_native": false, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM (SELECT * FROM __source(\"raw__orders\")) AS o JOIN __source(\"raw__customers\") AS c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM (SELECT * FROM __source(\"raw__orders\")) AS o JOIN __source(\"raw__customers\") AS c ON o.id = c.email\n" } } ], @@ -1435,7 +1435,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT * FROM __source(\"raw__orders\")) SELECT qty FROM x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH x AS (SELECT * FROM __source(\"raw__orders\")) SELECT qty FROM x\n" } }, { @@ -1445,7 +1445,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (qty) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * EXCLUDE (qty) FROM __source(\"raw__orders\")\n" } }, { @@ -1455,7 +1455,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > 5\n" } }, { @@ -1465,7 +1465,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __source(\"raw__orders\")\n" } }, { @@ -1475,7 +1475,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.id = c.email\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.id = c.email\n" } }, { @@ -1485,7 +1485,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1495,7 +1495,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status > 5\n" } }, { @@ -1505,7 +1505,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status\n" } }, { @@ -1515,7 +1515,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN quantity > 1 THEN quantity ELSE status END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN quantity > 1 THEN quantity ELSE status END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1525,7 +1525,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COALESCE(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1535,7 +1535,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity IN ('a', 'b')\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity IN ('a', 'b')\n" } }, { @@ -1545,7 +1545,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity LIKE '1%'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity LIKE '1%'\n" } }, { @@ -1555,7 +1555,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day', status) AS d FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_TRUNC('day', status) AS d FROM __source(\"raw__orders\")\n" } }, { @@ -1565,7 +1565,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at + status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1575,7 +1575,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LENGTH(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LENGTH(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1585,7 +1585,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at = 'not a date'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at = 'not a date'\n" } }, { @@ -1595,7 +1595,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity - ordered_at AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity - ordered_at AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1605,7 +1605,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at BETWEEN 1 AND 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at BETWEEN 1 AND 5\n" } }, { @@ -1615,7 +1615,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE o.ordered_at > (SELECT MAX(email) FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE o.ordered_at > (SELECT MAX(email) FROM __source(\"raw__customers\"))\n" } }, { @@ -1625,7 +1625,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" } }, { @@ -1635,7 +1635,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(p.payment_method) OVER () AS s FROM __ref(\"stg_payments\") p\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(p.payment_method) OVER () AS s FROM __ref(\"stg_payments\") p\n" } }, { @@ -1645,7 +1645,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT UPPER(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1655,7 +1655,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status = TRUE\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status = TRUE\n" } }, { @@ -1665,7 +1665,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS INTEGER) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(ordered_at AS INTEGER) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1675,7 +1675,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE ordered_at > 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE ordered_at > 5\n" } }, { @@ -1685,7 +1685,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders\")\n" } }, { @@ -1695,7 +1695,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\")\n" } }, { @@ -1705,7 +1705,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") INTERSECT SELECT id, email FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") INTERSECT SELECT id, email FROM __source(\"raw__customers\")\n" } }, { @@ -1715,7 +1715,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_a_function(id) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT not_a_function(id) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1725,7 +1725,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_TRUNC('day') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1735,7 +1735,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(status, 2) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROUND(status, 2) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1745,7 +1745,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUBSTRING(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUBSTRING(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1755,7 +1755,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY 5\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY 5\n" } }, { @@ -1765,7 +1765,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" } }, { @@ -1775,7 +1775,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM raw_orders\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM raw_orders\n" } }, { @@ -1785,7 +1785,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST('2020-01-01' AS TIMESTAMP) > 5 AS x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST('2020-01-01' AS TIMESTAMP) > 5 AS x\n" } }, { @@ -1795,7 +1795,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -1815,7 +1815,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ordered_at) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(ordered_at) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1825,7 +1825,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT AVG(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1835,7 +1835,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(CAST(ordered_at AS DATE)) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(CAST(ordered_at AS DATE)) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1845,7 +1845,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity, id) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity, id) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1855,7 +1855,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT STDDEV(status) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT STDDEV(status) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1865,7 +1865,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __ref(\"stg_orders_typed\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(status) AS s FROM __ref(\"stg_orders_typed\")\n" } }, { @@ -1875,7 +1875,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(waffle_name) AS s FROM __seed(\"waffle_types\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(waffle_name) AS s FROM __seed(\"waffle_types\")\n" } }, { @@ -1885,7 +1885,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE status) AS n FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COUNT(*) FILTER (WHERE status) AS n FROM __source(\"raw__orders\")\n" } }, { @@ -1895,7 +1895,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROW_NUMBER() AS rn FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROW_NUMBER() AS rn FROM __source(\"raw__orders\")\n" } }, { @@ -1905,7 +1905,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity, status) OVER (ORDER BY id) AS l FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LAG(quantity, status) OVER (ORDER BY id) AS l FROM __source(\"raw__orders\")\n" } }, { @@ -1915,7 +1915,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NTILE('a') OVER (ORDER BY id) AS t FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NTILE('a') OVER (ORDER BY id) AS t FROM __source(\"raw__orders\")\n" } }, { @@ -1925,7 +1925,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER (ORDER BY status RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity) OVER (ORDER BY status RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1935,7 +1935,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER w AS s FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(quantity) OVER w AS s FROM __source(\"raw__orders\")\n" } }, { @@ -1945,7 +1945,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY quantity > 1\n" } }, { @@ -1955,7 +1955,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ABS(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ABS(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1965,7 +1965,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ROUND(ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1975,7 +1975,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT EXTRACT(year FROM status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT EXTRACT(year FROM status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1985,7 +1985,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_DIFF('day', status, ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT DATE_DIFF('day', status, ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -1995,7 +1995,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SPLIT_PART(quantity, ',', 1) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SPLIT_PART(quantity, ',', 1) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2005,7 +2005,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT REGEXP_MATCHES(quantity, 'a') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT REGEXP_MATCHES(quantity, 'a') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2015,7 +2015,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SQRT(status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SQRT(status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2025,7 +2025,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT GREATEST(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT GREATEST(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2035,7 +2035,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEAST(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LEAST(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2045,7 +2045,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT NULLIF(quantity, status) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT NULLIF(quantity, status) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2055,7 +2055,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT IFNULL(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT IFNULL(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2065,7 +2065,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT STRFTIME(quantity, '%Y') AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT STRFTIME(quantity, '%Y') AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2075,7 +2075,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LOWER(ordered_at) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LOWER(ordered_at) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2085,7 +2085,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEFT(quantity, 2) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT LEFT(quantity, 2) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2095,7 +2095,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE() AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COALESCE() AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2105,7 +2105,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE is_valid(status)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE is_valid(status)\n" } }, { @@ -2115,7 +2115,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_an_agg(quantity) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT not_an_agg(quantity) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2125,7 +2125,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status OR quantity > 1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status OR quantity > 1\n" } }, { @@ -2135,7 +2135,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at * 2 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at * 2 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2145,7 +2145,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status % 2 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status % 2 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2155,7 +2155,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status IS TRUE\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status IS TRUE\n" } }, { @@ -2165,7 +2165,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at LIKE '2026%'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at LIKE '2026%'\n" } }, { @@ -2175,7 +2175,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity BETWEEN 'a' AND 'b'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity BETWEEN 'a' AND 'b'\n" } }, { @@ -2185,7 +2185,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN status THEN 1 END AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CASE WHEN status THEN 1 END AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2195,7 +2195,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT -status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT -status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2205,7 +2205,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at - status AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at - status AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2215,7 +2215,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING status\n" } }, { @@ -2225,7 +2225,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(quantity AS NOT_A_TYPE) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(quantity AS NOT_A_TYPE) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2235,7 +2235,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(status = 'x' AS TIMESTAMP) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(status = 'x' AS TIMESTAMP) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2245,7 +2245,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS BOOLEAN) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT CAST(ordered_at AS BOOLEAN) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2255,7 +2255,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > DATE '2026-13-45'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > DATE '2026-13-45'\n" } }, { @@ -2265,7 +2265,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + INTERVAL 'banana' AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT ordered_at + INTERVAL 'banana' AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2275,7 +2275,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT id, email FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT id, email FROM __source(\"raw__customers\"))\n" } }, { @@ -2285,7 +2285,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT (SELECT id, email FROM __source(\"raw__customers\") LIMIT 1) AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT (SELECT id, email FROM __source(\"raw__customers\") LIMIT 1) AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2295,7 +2295,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at IN (SELECT email FROM __source(\"raw__customers\"))\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at IN (SELECT email FROM __source(\"raw__customers\"))\n" } }, { @@ -2305,7 +2305,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE (id, customer_id) = (1, 2, 3)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE (id, customer_id) = (1, 2, 3)\n" } }, { @@ -2315,7 +2315,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") o ON o.id = o.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") o ON o.id = o.id\n" } }, { @@ -2325,7 +2325,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM (SELECT id FROM __source(\"raw__orders\")) a JOIN (SELECT email AS id FROM __source(\"raw__customers\")) b USING (id)\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * FROM (SELECT id FROM __source(\"raw__orders\")) a JOIN (SELECT email AS id FROM __source(\"raw__customers\")) b USING (id)\n" } }, { @@ -2335,7 +2335,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" } }, { @@ -2345,7 +2345,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" } }, { @@ -2355,7 +2355,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")), a AS (SELECT id FROM __source(\"raw__customers\")) SELECT id FROM a\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")), a AS (SELECT id FROM __source(\"raw__customers\")) SELECT id FROM a\n" } }, { @@ -2365,7 +2365,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) SELECT n FROM r\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) SELECT n FROM r\n" } }, { @@ -2375,7 +2375,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM b), b AS (SELECT id FROM __source(\"raw__orders\")) SELECT id FROM a\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM b), b AS (SELECT id FROM __source(\"raw__orders\")) SELECT id FROM a\n" } }, { @@ -2385,7 +2385,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\") ORDER BY status\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\") ORDER BY status\n" } }, { @@ -2395,7 +2395,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") EXCEPT SELECT id FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") EXCEPT SELECT id FROM __source(\"raw__customers\")\n" } }, { @@ -2405,7 +2405,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT created_at FROM __source(\"raw__customers\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT created_at FROM __source(\"raw__customers\")\n" } }, { @@ -2415,7 +2415,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.* FROM __source(\"raw__orders\") o\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT z.* FROM __source(\"raw__orders\") o\n" } }, { @@ -2425,7 +2425,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT * REPLACE (1 AS qty) FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT * REPLACE (1 AS qty) FROM __source(\"raw__orders\")\n" } }, { @@ -2435,7 +2435,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT COLUMNS('^nothing_') FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT COLUMNS('^nothing_') FROM __source(\"raw__orders\")\n" } }, { @@ -2445,7 +2445,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.qty FROM (SELECT * FROM __ref(\"stg_orders\")) x\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT x.qty FROM (SELECT * FROM __ref(\"stg_orders\")) x\n" } }, { @@ -2455,7 +2455,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT 'x'\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT 'x'\n" } }, { @@ -2465,7 +2465,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT -1\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT -1\n" } }, { @@ -2475,7 +2475,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT quantity\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT quantity\n" } }, { @@ -2485,7 +2485,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status) + 1 AS x FROM __source(\"raw__orders\")\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status) + 1 AS x FROM __source(\"raw__orders\")\n" } }, { @@ -2495,7 +2495,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")('abc') t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")('abc') t\n" } }, { @@ -2505,7 +2505,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(t.waffle_name) AS s FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT SUM(t.waffle_name) AS s FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" } }, { @@ -2515,7 +2515,7 @@ "expected_codes": [], "pending_native": true, "repo_files": { - "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT UPPER(@cents_to_dollars('p.amount_cents')) AS d FROM __ref(\"stg_payments\") p\n" + "models/marts/probe_m.sql": "MODEL (description \"Test model probe_m.\", materialized table);\n\nSELECT UPPER(@cents_to_dollars('p.amount_cents')) AS d FROM __ref(\"stg_payments\") p\n" } }, { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql index 3b2df3811b..338a02835f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql @@ -1,4 +1,5 @@ MODEL ( + description "Test model typed_orders.", materialized view, contract enforced, columns ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index d0d28c6689..d3e3b8b57c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1162,7 +1162,7 @@ def _with_chain_generated_logic(*, sql: str, target_bytes: int) -> str: def _base_model_sql() -> str: return "\n".join( ( - "MODEL (materialized view);", + "MODEL (description 'Test model.', materialized view);", "", "SELECT", " 0 AS id,", @@ -1178,7 +1178,7 @@ def _base_model_sql() -> str: def _chain_model_sql(*, index: int) -> str: previous_model: str = f"model_{index - 1:05d}" - return f'''MODEL (materialized view); + return f'''MODEL (description "Test model.", materialized view); WITH base AS ( SELECT @@ -1242,7 +1242,7 @@ def _chain_model_sql(*, index: int) -> str: def _test_heavy_base_model_sql(*, index: int) -> str: - return f"""MODEL (materialized view); + return f"""MODEL (description "Test model.", materialized view); SELECT {index} AS id, @@ -1253,7 +1253,7 @@ def _test_heavy_base_model_sql(*, index: int) -> str: def _test_heavy_chain_model_sql(*, index: int) -> str: previous_model: str = f"model_{index - 1:05d}" - return f'''MODEL (materialized view); + return f'''MODEL (description "Test model.", materialized view); WITH transformed AS ( SELECT @@ -1459,6 +1459,7 @@ def _layered_write_sources(*, project_dir: Path, source_count: int) -> None: sources_dir.mkdir() entries: str = "\n".join( f""" - name: source_{index:05d} + description: Test source source_{index:05d}. expression: "(SELECT {index} AS id, CAST({index} AS DOUBLE) AS amount, 'source' AS status)" columns: - name: id @@ -1477,6 +1478,7 @@ def _layered_write_seeds(*, project_dir: Path, seed_count: int) -> None: seeds_dir.mkdir() schema_entries: str = "\n".join( f""" - name: seed_{index:05d} + description: Test seed seed_{index:05d}. columns: - name: id type: INTEGER @@ -1497,7 +1499,7 @@ def _layered_write_functions(*, project_dir: Path, function_count: int) -> None: functions_dir.mkdir(parents=True) for index in range(function_count): (functions_dir / f"fn_{index:05d}.sql").write_text( - """FUNCTION ( + """FUNCTION (description "Test function.", arguments (input_value DOUBLE), returns DOUBLE, ); @@ -1566,7 +1568,7 @@ def _layered_write_schemas(*, project_dir: Path) -> None: ) schemas_dir.mkdir(parents=True) (schemas_dir / "benchmark_row.sql").write_text( - """SCHEMA ( + """SCHEMA (description "Test schema.", name benchmark_row, columns ( id (type INTEGER, nullable false), @@ -1626,14 +1628,14 @@ def _layered_model_header(*, index: int, audit_count: int) -> str: id (audits [not_null]), ), );""" - audit_header: str = """MODEL ( + audit_header: str = """MODEL (description "Test model.", columns ( id (nullable false, audits [not_null]), ), );""" return { (True, False): contract_header, - (False, True): "MODEL ();", + (False, True): "MODEL (description 'Test model.');", (False, False): audit_header, }[(index == 0, index >= audit_count)] @@ -1838,7 +1840,11 @@ def _semantic_model_header( audit_sql: str = {False: "", True: ", audits [not_null]"}[column_index < model_audit_count] nullable_sql: str = {False: "", True: ", nullable false"}[column_index == 0] declarations.append(f" {name} (type {data_type}{nullable_sql}{audit_sql})") - return "MODEL (\n columns (\n" + ",\n".join(declarations) + "\n ),\n);" + return ( + 'MODEL (\n description "Generated benchmark model.",\n columns (\n' + + ",\n".join(declarations) + + "\n ),\n);" + ) def _semantic_regular_model_sql( @@ -1958,7 +1964,7 @@ def _semantic_set_operation_model_sql( SHARED_DIAMOND_ROLLUP: str = "shared_orders_rollup" _SHARED_DIAMOND_FOLDER: str = "intermediate/shared_orders" _SHARED_DIAMOND_UPSTREAM_MODEL: str = f"model_{_SPINE_DEPTH - 1:05d}" -_SHARED_DIAMOND_HEADER: str = """MODEL ( +_SHARED_DIAMOND_HEADER: str = """MODEL (description "Test model.", columns ( id (type INTEGER, nullable false), amount (type DOUBLE), @@ -2178,16 +2184,19 @@ def build_rule_gated_test_project_files(*, filler_model_count: int) -> dict[str, ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_refunds\n schema: main\n table: raw_refunds\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_refunds\n description: Test source raw_refunds.\n" + " schema: main\n table: raw_refunds\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n' ), "models/bad_star.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model bad_star.", ' + 'materialized view);\n\nSELECT * FROM __source("raw_orders")\n' ), "tests/unit/orders_case.sql": ( 'TEST (name "orders_case");\n' @@ -2199,7 +2208,8 @@ def build_rule_gated_test_project_files(*, filler_model_count: int) -> dict[str, } for index in range(filler_model_count): files[f"models/filler/filler_{index:03d}.sql"] = ( - f"MODEL (materialized view);\n\nSELECT {index} AS filler_id\n" + "MODEL (description 'Test model.', " + f"materialized view);\n\nSELECT {index} AS filler_id\n" ) return files @@ -2224,23 +2234,24 @@ def build_empty_input_test_project_files( f"[rules.rule_options.SQBRTEST203]\nallowed_tests = {allowed_tests_toml}\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" "SELECT order_id, amount * 2 AS doubled_amount\n" 'FROM __source("raw_orders")\nWHERE amount > 0\n' ), "models/large_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model large_orders.', materialized view);\n\n" 'SELECT order_id FROM __source("raw_orders") WHERE amount > 100\n' ), "models/customers.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model customers.', materialized view);\n\n" 'SELECT order_id AS customer_id FROM __source("raw_orders") WHERE amount > 10\n' ), "models/order_summary.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model order_summary.', materialized view);\n\n" "SELECT COUNT(*) AS order_count, COALESCE(SUM(amount), 0) AS total_amount\n" 'FROM __source("raw_orders")\n' ), @@ -2389,17 +2400,18 @@ def write_relation_stub_project( 'schema = "main"\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT CAST(7 AS INTEGER) AS order_id\n" + "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT CAST(7 AS INTEGER) AS order_id\n" ), "models/order_status.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_status.', materialized table);\n\n" f"WITH {stub_cte_name} AS (\n" " SELECT 'pending' AS order_id, 'pending' AS ghost_status\n" ")\n" f'SELECT {selected_column} FROM __ref("orders")\n' ), "models/next_order.sql": ( - "MODEL (\n" + "MODEL (description 'Test model next_order.',\n" " materialized table,\n" " contract enforced,\n" " columns (order_id (type INTEGER), next_order_id (type INTEGER)),\n" @@ -2423,7 +2435,7 @@ def write_relation_stub_project( "CONSTANT (name min_quantity, value 1);\n" ), "models/marts/channel_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model channel_orders.',\n" " materialized table,\n" " columns (\n" " order_id (nullable false, audits [not_null]),\n" @@ -2686,7 +2698,8 @@ def write_set_operation_project( } for model in models: files[f"models/{model.name}.sql"] = ( - f"MODEL (materialized view, schema analytics);\n\n{model.query_sql}\n" + "MODEL (description 'Test model.', materialized view, schema analytics);\n\n" + f"{model.query_sql}\n" ) prepare_inline_project( tmp_path=project_dir.parent, project_name=project_dir.name, repo_files=files @@ -2776,11 +2789,11 @@ def write_union_fixture_test_project( f'database = "{project_dir / "orders.duckdb"}"\n[rules]\nselect = []\n' ), "models/base_orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model base_orders.', materialized view);\n\n" "SELECT 1 AS order_id, CAST(1 AS DOUBLE) AS amount, 'open' AS status\n" ), "models/orders.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT b.order_id, b.amount, b.status FROM __ref("base_orders") AS b\n' ), } @@ -2823,3 +2836,55 @@ def measure_union_fixture_compile( ) ) return min(measurements) + + +REQUIRED_DESCRIPTIONS_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_REQUIRED_DESCRIPTIONS_ORDERS_MODEL: str = ( + 'MODEL (description "One row per order");\n\nSELECT 1 AS order_id\n' +) +_REQUIRED_DESCRIPTIONS_COMPILE: tuple[str, ...] = ("--no-color", "compile", "--json", "--no-cache") + + +def hooked_model_file(*, model_name: str, hook: str) -> tuple[str, str]: + """Return a described model file whose post hook references ``hook``.""" + + return ( + f"models/{model_name}.sql", + f'MODEL (description "Orders with a hook", post_hooks [{hook}]);\n\nSELECT 1 AS order_id\n', + ) + + +def python_node_source( + *, + module: str, + decorator: str, + name: str, + docstring_line: str = "", + extra_import: str = "", + decorator_arguments: str = "", +) -> str: + """Render one decorated Python node module for required-description E2Es.""" + + return ( + f"from sqlbuild.{module} import {decorator}\n{extra_import}\n\n" + f"@{decorator}{decorator_arguments}\ndef {name}(ctx):\n{docstring_line} return None\n" + ) + + +def compile_inline_files( + *, tmp_path: Path, files: tuple[tuple[str, str], ...] +) -> subprocess.CompletedProcess[str]: + """Compile a described orders project extended with ``files``.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": REQUIRED_DESCRIPTIONS_PROJECT, + "models/orders.sql": _REQUIRED_DESCRIPTIONS_ORDERS_MODEL, + **dict(files), + }, + ) + return run_sqb(project_dir=project_dir, command=_REQUIRED_DESCRIPTIONS_COMPILE) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index 0328b31d8f..ddf3c7d886 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -53,7 +53,7 @@ edit=lambda root: write_project_file( root, "models/marts/order_quantities.sql", - 'MODEL ();\n\nSELECT order_id, quantity FROM __ref("stg_orders")\n', + 'MODEL (description "Test model order_quantities.");\n\nSELECT order_id, quantity FROM __ref("stg_orders")\n', ), ), CompileCacheInvalidationTestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py index 378ac50db1..b0c58b50db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_complexity_guard.py @@ -45,7 +45,7 @@ def test_given_deep_trusted_model_when_compiling_then_sql_analysis_completes( 'schema = "dev"\n' ), "models/deep_expression.sql": ( - "MODEL (columns (resolved_value (type NUMBER)));\n\n" + "MODEL (description 'Test model deep_expression.', columns (resolved_value (type NUMBER)));\n\n" f"SELECT {nested_coalesce_expression(function_depth=test_case.function_depth)} " "AS resolved_value\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py index 8f8df2d770..18978d26f8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_selection.py @@ -68,12 +68,12 @@ def test_given_model_selection_when_compiling_then_limits_offline_analysis_and_r "[targets.dev]\n" 'schema = "dev"\n' ), - "models/order_record.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/order_record.sql": "MODEL (description 'Test model order_record.');\n\nSELECT 1 AS order_id\n", "models/result.sql": ( - 'MODEL (columns (order_id ()));\n\nSELECT order_id FROM __ref("order_record")\n' + 'MODEL (description "Test model result.", columns (order_id ()));\n\nSELECT order_id FROM __ref("order_record")\n' ), "models/unrelated.sql": ( - "MODEL (columns (missing_column ()));\n\nSELECT 1 AS present_column\n" + "MODEL (description 'Test model unrelated.', columns (missing_column ()));\n\nSELECT 1 AS present_column\n" ), "selectors.txt": "result\n", }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5dc59b4235..27cb152395 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -42,28 +42,28 @@ 1000, 14.0, 3 * _GIB // 2, - "cfb7bc162018ddc64f9d03ddd2c316e34a333d4129906d8e0b3310399604b980", + "6ea005eef07fe6d2ec47980bc573903efa060cfde55aa5488e63b29ce846280e", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, 45.0, 11 * _GIB // 4, - "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", + "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, 75.0, 13 * _GIB // 4, - "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", + "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, 145.0, 4 * _GIB, - "7a4b6cca4a2fd4b7f5961760177f8b18133854014eaf15a980e8283df646a2cf", + "913a08c2277304efacf75e293728767a3fa416f6b32f5488dca6ab5baea80668", ), ), ids=lambda case: case.description, @@ -141,8 +141,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ 22.0, 24.0, 11 * _GIB // 4, - "c35ea7ec71e6e81059146bc1cd72f10e6493248dc6274aba7737d616d434585b", - "255c19bb188491b53d44c2e5d41472137f763806c699f5acb3899c6b47c5360b", + "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", + "f7f8a3d881496664ec7e293144d93b8b8e486a04eba830d9cc6500d1b28147f3", 3, ), DenseWarmEditCompileGuardTestCase( @@ -152,8 +152,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ 36.0, 38.0, 13 * _GIB // 4, - "b07422282a6800de7830b9c4950f76bdc6fd555ebd3f21152b8649f976be74dc", - "c239f53fb5fa787d250f5f7bf404b3731b480b77d2690dd31fb41ba727475fc0", + "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", + "571fe58493234057c7128b4929d3518532731192f6fef85da0496788826bb172", 3, ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index aa0f9a0f67..4773daca5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -57,16 +57,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( - "f96a10c8d9d218c347432118e069f378140bb6f3a852676d968ad9d5f67d2589" + "7a1978e9f8871b2252abea5e8763c570305f97342b3b3f51e2a2992238cc3798" ), expected_leaf_edit_fingerprint=( - "1e3adee6c9d336de987ea15e3c054726cdf0a00ba8a2b4dcc2f0953b6711cf2c" + "a041d989ef3810e0840361a4f793548b428f941da2f265462a4b46a5bf4d7676" ), expected_macro_edit_fingerprint=( - "81eb4ccff1c67d77a1a4290a239007b92f21bd3112729f64a508592166ed3c39" + "8b62365b60a9f60afaffe6571d6849a187480d503a3114f8783050ebed46363b" ), expected_project_config_fingerprint=( - "c8fbec90f4c03c1bc8c13478121a2ae4c8b9d7c18b0270102130b689a2e76549" + "d077b4676e827b41383cb199f342867995ec1a5564e859e85558a24d1201abf6" ), macro_call_interval=6, scoped_macros=True, @@ -91,16 +91,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( - "26de3005daf1b44463b25165238e83c9a8085b00d942eb4beb1b735ce5f09b57" + "d293f1e9dbe2787d21fa6e5f20c79206ad9beea1e3be63299ac0a5a1e3464ffd" ), expected_leaf_edit_fingerprint=( - "e032f1c0edf2e767f356a6af4891c4dcde0d214d6f691ea1dd0336968e97b707" + "a2e0a328277276f960bef73b4fbc85f9dcacbfad73c10af2df129b78592606ba" ), expected_macro_edit_fingerprint=( - "9e20327a775d2f876f25621d60f8bb81b62d529823a61b8d9686f86de322790c" + "38779c13ac071bf082455f27cdc1d071060da243b9a3d47f0f34b384962b5d11" ), expected_project_config_fingerprint=( - "b65db3cfba6cccef479df505832f7e5821f03b7a0b31e8de967b14796644c768" + "a26f85f97e7c28d40056f95d6be646399b03b215f0a24884f4c26b7b3e996595" ), macro_call_interval=6, scoped_macros=True, @@ -125,16 +125,16 @@ expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( - "b4e5e14d9c1a7ac2e5fc54eac8d1ff2874f2e3042952cb4b80c68c4d4bd1503c" + "8994dd1e87cb97312d492ea66731e4f01f4da08dc2680dd6814f1e4bba23a775" ), expected_leaf_edit_fingerprint=( - "fa0b24b685c7f42a14588ab09e7c929332295ecc83cc9adfdc589b4cbc1c6128" + "f1f25d949e182901656fcef88c7c566254c847174b888bd35c87372aea347807" ), expected_macro_edit_fingerprint=( - "39079f601b624ba4d13d889e9a228c9713ec7ed09ed2127afd3b2c2b48465fbb" + "b8f3cc27552a0d49e97b112732a3b6331040f0dcdec6ed15cc7058a21e98aa4f" ), expected_project_config_fingerprint=( - "6dd7ae799420dcb8a49190356a0b97253b59a3106502cc8cf242ee75c2aedaa2" + "ff3467ff64bfbfa1073a169d53576bfee8699c2529a37250c44e6cc2ecd11845" ), macro_call_interval=6, scoped_macros=True, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index f9e223e1c3..eac764b865 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -39,7 +39,7 @@ expected_max_wall_seconds=17.0, expected_max_rss_bytes=int(1.5 * _GIB), expected_semantic_fingerprint=( - "b763ab23427ed273feb5f80b0b03cf084c7af2f1216500bc5ad7db0be23c2ac1" + "24d42df4968ff361ac0e07e2dbe03abd13bc6948b6812b8188f10d7b0aa9fef4" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -54,7 +54,7 @@ expected_max_wall_seconds=25.0, expected_max_rss_bytes=int(1.75 * _GIB), expected_semantic_fingerprint=( - "fc03f9e22fcf029217e77d3c6e18c9d9945e98e9425a5420a9c4756219a8fd77" + "4fd79eff9a628608a2f9836e1325cec58f9f3d5ac870e284c734f09c505c700f" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -69,7 +69,7 @@ expected_max_wall_seconds=50.0, expected_max_rss_bytes=2 * _GIB, expected_semantic_fingerprint=( - "34e21096cdc9c3a34655f67660de9775b0ac6cebfb5275ce2d647f245ce23f79" + "d83928b7553bcc58564eabc4669045a63e5b314726672ada0a3d979e54ab8573" ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py index c3a809b4ae..cfcd54bb24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_header_keys.py @@ -21,11 +21,11 @@ repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "functions/sql/normalize_status.sql": ( - "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + "FUNCTION (description 'Test function normalize_status.',\n arguments (status VARCHAR),\n returns VARCHAR,\n" " replay_on_change full,\n);\n\nlower(status)\n" ), "models/order_statuses.sql": ( - "MODEL (materialized table);\n\nSELECT __udf(\"normalize_status\")('OPEN') " + "MODEL (description 'Test model order_statuses.', materialized table);\n\nSELECT __udf(\"normalize_status\")('OPEN') " "AS status\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py index 38351db4a1..de78810c9e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py @@ -15,7 +15,7 @@ from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb _STARTS_WITH_FUNCTION: str = ( - "FUNCTION (\n" + "FUNCTION (description 'Test function.',\n" " arguments (value STRING, prefix STRING),\n" " returns BOOLEAN,\n" " database warehouse,\n" @@ -125,7 +125,7 @@ def test_given_builtin_spelling_when_compiling_then_dialect_support_is_enforced( "sqlbuild_project.toml": test_case.project_toml, "functions/sql/starts_with.sql": _STARTS_WITH_FUNCTION, "models/products.sql": ( - f"MODEL (database warehouse, schema analytics);\n\n{test_case.query_sql}" + f"MODEL (description 'Test model products.', database warehouse, schema analytics);\n\n{test_case.query_sql}" ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py index 35980a6788..ecfeab998f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py @@ -34,6 +34,7 @@ """ seeds: - name: countries + description: Test seed countries. columns: - name: code type: VARCHAR @@ -52,11 +53,11 @@ "from sqlbuild.functions import udf\n\n" '@udf(arguments={}, returns="INTEGER", runtime_version="3.11")\n' "def main():\n" - " return 42\n" + " '''Test function main.'''\n return 42\n" ) _MANAGED_SOURCE: str = ( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -67,7 +68,7 @@ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_events.'''\n return [{'event_id': 1}]\n" ) @@ -78,7 +79,7 @@ description="model without logical schema fails under preserve", repo_files={ "sqlbuild_project.toml": _PRESERVE_CONFIG, - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Model 'orders' has no logical schema", @@ -97,7 +98,7 @@ description="SQL scalar function without logical schema fails under preserve", repo_files={ "sqlbuild_project.toml": _PRESERVE_CONFIG, - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", }, expected_exit_code=1, expected_stderr_fragment="SQL function 'answer' has no logical schema", @@ -108,7 +109,7 @@ "sqlbuild_project.toml": _PRESERVE_CONFIG, "functions/sql/answers.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function answers.", returns table ( answer INTEGER ) @@ -132,6 +133,7 @@ @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -174,16 +176,17 @@ def main(): """ ).strip() + "\n", - "models/marts/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "functions/python/python_answer.py": dedent( """ from sqlbuild.functions import udf @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -198,16 +201,17 @@ def main(): "sqlbuild_project.toml": _PRESERVE_CONFIG.replace( 'schema = "preserve"', 'schema = "physical"' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "functions/python/python_answer.py": dedent( """ from sqlbuild.functions import udf @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 42 """ ).strip() @@ -220,7 +224,7 @@ def main(): description="warehouse model without explicit schema fails offline", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Model 'orders' has no explicitly resolved physical write schema", @@ -239,7 +243,7 @@ def main(): description="warehouse SQL scalar function without explicit schema fails offline", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", }, expected_exit_code=1, expected_stderr_fragment="Function 'answer' has no explicitly resolved physical write schema", @@ -249,7 +253,7 @@ def main(): repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, "functions/sql/answers.sql": ( - "FUNCTION (returns table (answer INTEGER));\n\nSELECT 42 AS answer\n" + "FUNCTION (description 'Test function answers.', returns table (answer INTEGER));\n\nSELECT 42 AS answer\n" ), }, expected_exit_code=1, @@ -280,7 +284,7 @@ def main(): description="resource schema supplies physical namespace", repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -291,7 +295,7 @@ def main(): "sqlbuild_project.toml": ( _POSTGRES_CONFIG_WITHOUT_SCHEMA + '\n[defaults]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -305,7 +309,7 @@ def main(): ) + '\n[targets.dev]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -316,7 +320,7 @@ def main(): "sqlbuild_project.toml": ( _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA + 'schema = "preserve"\n' ), - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -325,7 +329,7 @@ def main(): description="named target rejects model logical schema as strategy", repo_files={ "sqlbuild_project.toml": _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA, - "models/orders.sql": "MODEL (schema analytics);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', schema analytics);\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment=( @@ -338,7 +342,7 @@ def main(): "sqlbuild_project.toml": ( _NAMED_POSTGRES_CONFIG_WITHOUT_SCHEMA + '\n[defaults]\nschema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Named target 'dev' must explicitly set schema", @@ -350,7 +354,7 @@ def main(): 'database = "analytics"', 'database = "analytics"\nschema = "connection_schema"', ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment="Named target 'dev' must explicitly set schema", @@ -361,7 +365,7 @@ def main(): "sqlbuild_project.toml": ( _POSTGRES_CONFIG_WITHOUT_SCHEMA + 'schema = "analytics"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -389,10 +393,10 @@ def main(): "[connection]\n" 'database = ":memory:"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", "seeds/schema.yml": _SEED_SCHEMA, "seeds/countries.csv": "code\nGB\n", - "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", + "functions/sql/answer.sql": "FUNCTION (description 'Test function answer.', returns INTEGER);\n\n42\n", "sources/raw.yml": _MANAGED_SOURCE, "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, @@ -408,7 +412,7 @@ def main(): 'database = ":memory:"\n\n' "[targets.dev]\n" ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py index 31027c2ed9..07e4f9ad95 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_path_defaults.py @@ -37,7 +37,7 @@ """ ).strip() + "\n", - "models/commerce/partner_feed/staging/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/commerce/partner_feed/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=0, expected_stderr_fragment="", @@ -61,7 +61,7 @@ """ ).strip() + "\n", - "models/commerce/eu/staging/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/commerce/eu/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, expected_exit_code=1, expected_stderr_fragment=( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index b01925c069..1df9bffb70 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -169,7 +169,7 @@ def test_given_python_under_unsupported_root_when_compiling_then_command_fails_w description="custom Rule harness test path", repo_files={ "sqlbuild_project.toml": 'name = "rule_harness_layout"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", "tests/rules/test_order_policy.py": "def test_order_policy(): pass\n", }, expected_exit_code=0, @@ -215,7 +215,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -225,7 +225,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte " type: VARCHAR\n" ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" 'SELECT order_id, status FROM __source("raw_orders")\n' ), "python/_helpers.py": ( @@ -245,14 +245,14 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return order_rows()\n" + " '''Test loader raw_orders.'''\n return order_rows()\n" ), "python/orders/exports/assets.py": ( "from sqlbuild.assets import asset\n" "from sqlbuild.refs import model\n\n\n" "@asset(depends_on=model('fact_orders'))\n" "def orders_export(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" + " '''Test asset orders_export.'''\n relation = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()\n" " return ctx.result(payload={'order_count': int(rows[0][0])})\n" ), @@ -261,7 +261,7 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte "from sqlbuild.checks import check\n\n\n" "@check(depends_on=orders_export)\n" "def orders_export_not_empty(ctx):\n" - " payload = ctx.result_of(node_function=orders_export).payload\n" + " '''Test check orders_export_not_empty.'''\n payload = ctx.result_of(node_function=orders_export).payload\n" " if payload['order_count'] <= 0:\n" " return ctx.fail(message='orders export is empty')\n" " return ctx.pass_(message='orders export is ready')\n" @@ -322,7 +322,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return None\n\n\n" + " '''Test task orders.'''\n return None\n\n\n" "@factory\n" "def order_nodes():\n" " return orders\n" @@ -342,7 +342,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi " global orders\n\n" " @task\n" " def orders(ctx):\n" - " return None\n\n" + " '''Test task orders.'''\n return None\n\n" " return orders\n" ), }, @@ -386,12 +386,12 @@ def test_given_factory_returning_module_node_when_planning_then_node_registers_o "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return None\n\n\n" + " '''Test task orders.'''\n return None\n\n\n" "@factory\n" "def order_nodes():\n" " @task(name='orders')\n" " def generated_orders(ctx):\n" - " return None\n" + " '''Test task generated_orders.'''\n return None\n" " return generated_orders\n" ), }, @@ -441,7 +441,7 @@ def test_given_factory_returning_distinct_same_name_node_when_planning_then_conf "from sqlbuild.tasks import task\n\n\n" "@task\n" "def orders(ctx):\n" - " return ctx.result(payload={'status': python.helpers.clean.label()})\n" + " '''Test task orders.'''\n return ctx.result(payload={'status': python.helpers.clean.label()})\n" ), }, expected_build_fragments=("orders", "\u2713 Completed successfully"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py new file mode 100644 index 0000000000..8c5edfcebc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -0,0 +1,669 @@ +"""E2E coverage of the P010 compile error for named resources without a description.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import asdict +from itertools import chain +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.compiler.compile.constants import DESCRIPTION_REQUIRED_INPUT_KINDS +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FormatDescriptionResolutionE2ECase, + PathDefaultDescriptionCase, + RequiredDescriptionAggregateCase, + RequiredDescriptionCase, + RequiredDescriptionPlanCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + REQUIRED_DESCRIPTIONS_PROJECT, + compile_inline_files, + hooked_model_file, + python_node_source, + require_sql_analysis_output, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_SEED_CSV: str = "product_type_id,label\n1,waffle\n" +_SEED_COLUMNS: str = ( + " columns:\n - name: product_type_id\n type: INTEGER\n" + " - name: label\n type: VARCHAR\n" +) +_ORDER_TOTALS_MODEL: tuple[str, str] = ( + "models/order_totals.sql", + 'MODEL (description "Order count");\n\nSELECT COUNT(*) AS order_count FROM __ref("orders")\n', +) +_SCENARIO_BODY: str = ( + "WITH\n__ref__orders AS (\n SELECT 1 AS order_id\n),\n" + "__expected__order_totals AS (\n SELECT 1 AS order_count\n)\nSELECT 1\n" +) + + +_PROVIDER_DESCRIBED: str = ( + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n" + ' """Client for the orders API."""\n\n' + ' base_url: str = "https://example.com"\n' +) +_PROVIDER_UNDESCRIBED: str = ( + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n" + ' base_url: str = "https://example.com"\n' +) + + +_CASES: tuple[RequiredDescriptionCase, ...] = ( + RequiredDescriptionCase( + description="model header", + missing_files=( + ("models/order_totals.sql", "MODEL (materialized table);\nSELECT 1 AS id\n"), + ), + described_files=( + ( + "models/order_totals.sql", + 'MODEL (description "Order totals", materialized table);\nSELECT 1 AS id\n', + ), + ), + expected_diagnostic=("P010", "models/order_totals.sql", 1), + expected_message="model 'order_totals' has no description", + expected_help_fragments=( + "to describe model 'order_totals', add this to the MODEL header in " + "models/order_totals.sql:\n" + " MODEL (\n" + ' description "What one row of order_totals represents",\n' + " ...\n" + " );", + ), + ), + RequiredDescriptionCase( + description="scenario header", + missing_files=( + ( + "tests/scenarios/orders__single_order.sql", + "SCENARIO ();\n\n" + _SCENARIO_BODY, + ), + _ORDER_TOTALS_MODEL, + ), + described_files=( + ( + "tests/scenarios/orders__single_order.sql", + 'SCENARIO (description "One order produces one row");\n\n' + _SCENARIO_BODY, + ), + _ORDER_TOTALS_MODEL, + ), + expected_diagnostic=("P010", "tests/scenarios/orders__single_order.sql", 1), + expected_message="scenario 'orders__single_order' has no description", + expected_help_fragments=( + "add this to the SCENARIO header in tests/scenarios/orders__single_order.sql:\n" + " SCENARIO (\n" + ' description "The behaviour orders__single_order proves",', + ), + ), + RequiredDescriptionCase( + description="seed YAML entry", + missing_files=( + ("seeds/product_types.csv", _SEED_CSV), + ("seeds/product_types.yml", "seeds:\n - name: product_types\n" + _SEED_COLUMNS), + ), + described_files=( + ("seeds/product_types.csv", _SEED_CSV), + ( + "seeds/product_types.yml", + "seeds:\n - name: product_types\n description: Product type catalogue\n" + + _SEED_COLUMNS, + ), + ), + expected_diagnostic=("P010", "seeds/product_types.yml", 2), + expected_message="seed 'product_types' has no description", + expected_help_fragments=( + "add `description` to its entry in seeds/product_types.yml:\n" + " - name: product_types\n" + " description: What one row of product_types represents", + ), + ), + RequiredDescriptionCase( + description="source YAML entry", + missing_files=( + ( + "sources/raw.yml", + "sources:\n - name: raw_customers\n schema: main\n table: customers\n", + ), + ), + described_files=( + ( + "sources/raw.yml", + "sources:\n - name: raw_customers\n description: Customer records\n" + " schema: main\n table: customers\n", + ), + ), + expected_diagnostic=("P010", "sources/raw.yml", 2), + expected_message="source 'raw_customers' has no description", + expected_help_fragments=( + "add `description` to its entry in sources/raw.yml:\n" + " - name: raw_customers\n", + ), + ), + RequiredDescriptionCase( + description="managed source described by its loader docstring", + missing_files=( + ("sources/raw.yml", "sources:\n - name: raw_customers\n managed: true\n"), + ( + "python/loaders/raw_customers.py", + python_node_source(module="loaders", decorator="loader", name="raw_customers"), + ), + ), + described_files=( + ("sources/raw.yml", "sources:\n - name: raw_customers\n managed: true\n"), + ( + "python/loaders/raw_customers.py", + python_node_source( + module="loaders", + decorator="loader", + name="raw_customers", + docstring_line=' """Customer records from the CRM export."""\n', + ), + ), + ), + expected_diagnostic=("P010", "sources/raw.yml", 2), + expected_message="source 'raw_customers' has no description", + expected_help_fragments=( + "or describe its loader 'raw_customers' in python/loaders/raw_customers.py with a " + 'docstring or @loader(description="What one row of raw_customers represents")', + ), + ), + RequiredDescriptionCase( + description="SQL function header", + missing_files=( + ( + "functions/sql/is_large_order.sql", + "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\n" + "amount > 100\n", + ), + ), + described_files=( + ( + "functions/sql/is_large_order.sql", + 'FUNCTION (\n description "Whether an order amount is large",\n' + " arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/sql/is_large_order.sql", 1), + expected_message="function 'is_large_order' has no description", + expected_help_fragments=( + "add this to the FUNCTION header in functions/sql/is_large_order.sql:\n" + " FUNCTION (\n" + ' description "What is_large_order returns",', + ), + ), + RequiredDescriptionCase( + description="table function header", + missing_files=( + ( + "functions/sql/order_lines.sql", + "FUNCTION (\n arguments (p_order_id INTEGER),\n" + " returns table (order_id INTEGER),\n);\n\nSELECT p_order_id AS order_id\n", + ), + ), + described_files=( + ( + "functions/sql/order_lines.sql", + 'FUNCTION (\n description "Lines of one order",\n' + " arguments (p_order_id INTEGER),\n" + " returns table (order_id INTEGER),\n);\n\nSELECT p_order_id AS order_id\n", + ), + ), + expected_diagnostic=("P010", "functions/sql/order_lines.sql", 1), + expected_message="function 'order_lines' has no description", + expected_help_fragments=('description "What order_lines returns"',), + ), + RequiredDescriptionCase( + description="Python UDF described by its docstring", + missing_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + described_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n" + ' """Whether an order amount is large."""\n' + " return amount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/python/is_large_order_py.py", 4), + expected_message="function 'is_large_order_py' has no description", + expected_help_fragments=( + "give its function in functions/python/is_large_order_py.py a docstring or add " + "this to @udf:\n" + ' @udf(description="What is_large_order_py returns", ...)', + ), + ), + RequiredDescriptionCase( + description="Python UDF described by its decorator", + missing_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + described_files=( + ( + "functions/python/is_large_order_py.py", + "from sqlbuild.functions import udf\n\n\n" + "@udf(\n" + ' description="Whether an order amount is large",\n' + ' arguments={"amount": "INTEGER"},\n' + ' returns="BOOLEAN",\n' + ' runtime_version="3.11",\n' + ")\n" + "def main(amount: int) -> bool:\n return amount > 100\n", + ), + ), + expected_diagnostic=("P010", "functions/python/is_large_order_py.py", 4), + expected_message="function 'is_large_order_py' has no description", + expected_help_fragments=(), + ), + RequiredDescriptionCase( + description="named SQL hook header", + missing_files=( + ("hooks/sql/analyze_orders.sql", "HOOK ();\n\nSELECT 1\n"), + hooked_model_file(model_name="analyzed_orders", hook='sql("analyze_orders")'), + ), + described_files=( + ( + "hooks/sql/analyze_orders.sql", + 'HOOK (description "What analyze_orders does");\n\nSELECT 1\n', + ), + hooked_model_file(model_name="analyzed_orders", hook='sql("analyze_orders")'), + ), + expected_diagnostic=("P010", "hooks/sql/analyze_orders.sql", 1), + expected_message="hook 'analyze_orders' has no description", + expected_help_fragments=( + "add this to the HOOK header in hooks/sql/analyze_orders.sql:\n" + ' HOOK (description "What analyze_orders does");', + ), + ), + RequiredDescriptionCase( + description="Python hook", + missing_files=( + ( + "hooks/python/notify_orders.py", + python_node_source(module="hooks", decorator="hook", name="notify_orders"), + ), + hooked_model_file(model_name="notified_orders", hook='python("notify_orders")'), + ), + described_files=( + ( + "hooks/python/notify_orders.py", + python_node_source( + module="hooks", + decorator="hook", + name="notify_orders", + docstring_line=' """Notify the orders channel."""\n', + ), + ), + hooked_model_file(model_name="notified_orders", hook='python("notify_orders")'), + ), + expected_diagnostic=("P010", "hooks/python/notify_orders.py", 4), + expected_message="hook 'notify_orders' has no description", + expected_help_fragments=( + "give its function in hooks/python/notify_orders.py a docstring or pass a " + "description to @hook:\n" + ' @hook(description="What notify_orders does", ...)', + ), + ), + RequiredDescriptionCase( + description="loader without a source declaration", + missing_files=( + ( + "python/loaders/raw_returns.py", + python_node_source(module="loaders", decorator="loader", name="raw_returns"), + ), + ), + described_files=( + ( + "python/loaders/raw_returns.py", + python_node_source( + module="loaders", + decorator="loader", + name="raw_returns", + docstring_line=' """Returned orders from the support desk."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/loaders/raw_returns.py", 4), + expected_message="loader 'raw_returns' has no description", + expected_help_fragments=('@loader(description="What raw_returns does", ...)',), + ), + RequiredDescriptionCase( + description="task", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source(module="tasks", decorator="task", name="refresh_exports"), + ), + ), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=' """Refresh the order exports."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' has no description", + expected_help_fragments=('@task(description="What refresh_exports does", ...)',), + ), + RequiredDescriptionCase( + description="asset", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source(module="assets", decorator="asset", name="order_report"), + ), + ), + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + docstring_line=' """Weekly order report file."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' has no description", + expected_help_fragments=('@asset(description="What order_report does", ...)',), + ), + RequiredDescriptionCase( + description="check", + missing_files=( + ( + "python/checks/orders_present.py", + python_node_source( + module="checks", + decorator="check", + name="orders_present", + extra_import="from sqlbuild.refs import model\n", + decorator_arguments='(depends_on=model("orders"))', + ), + ), + ), + described_files=( + ( + "python/checks/orders_present.py", + python_node_source( + module="checks", + decorator="check", + name="orders_present", + extra_import="from sqlbuild.refs import model\n", + decorator_arguments='(depends_on=model("orders"))', + docstring_line=' """Orders exist after the build."""\n', + ), + ), + ), + expected_diagnostic=("P010", "python/checks/orders_present.py", 5), + expected_message="check 'orders_present' has no description", + expected_help_fragments=('@check(description="What orders_present does", ...)',), + ), + RequiredDescriptionCase( + description="provider", + missing_files=(("providers/orders_api.py", _PROVIDER_UNDESCRIBED),), + described_files=(("providers/orders_api.py", _PROVIDER_DESCRIBED),), + expected_diagnostic=("P010", "providers/orders_api.py", 4), + expected_message="provider 'orders_api' has no description", + expected_help_fragments=( + "add a class docstring in providers/orders_api.py:\n" + " class OrdersApi(Provider):\n" + ' """What orders_api provides."""', + ), + ), +) + + +_PATH_DEFAULT_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n\n' + '[path_defaults.marts]\ndescription = "Mart model"\n' +) +_OWN_DESCRIPTION_MODEL: str = 'MODEL (description "Own description");\n\nSELECT 1 AS id\n' +_INHERITING_MODEL: str = "MODEL ();\n\nSELECT 1 AS id\n" + + +_TYPED_NULL_FIXTURE_FILES: tuple[tuple[str, str], ...] = ( + ("models/up.sql", 'MODEL (description "Upstream rows");\n\nSELECT 1 AS a\n'), + ( + "models/marts/a_model.sql", + 'MODEL (description "Own description");\n\nSELECT a\nFROM __ref("up")\n', + ), + ("models/marts/b_model.sql", _INHERITING_MODEL), + ( + "tests/unit/test_a_model.sql", + 'TEST (description "Null values pass through");\n\n' + "WITH __ref__up AS (\n SELECT CAST(NULL AS INTEGER) AS a\n),\n\n" + "__expected__a_model AS (\n SELECT CAST(NULL AS INTEGER) AS a\n)\n\nSELECT 1\n", + ), +) + + +_ALL_UNDESCRIBED_FILES: tuple[tuple[str, str], ...] = ( + *dict(chain.from_iterable(case.missing_files for case in _CASES)).items(), + ("models/undescribed_orders.sql", "MODEL (materialized view);\n\nSELECT 1 AS id\n"), +) + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionCase(**asdict(case)) for case in _CASES], + ids=lambda case: case.description, +) +def test_given_resource_without_description_when_compiling_then_reports_p010_with_snippet( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.missing_files + ) + + diagnostics, text = require_sql_analysis_output(result) + payload: dict[str, Any] = json.loads(result.stdout) + assert result.returncode == 1, result.stdout + result.stderr + assert diagnostics == (test_case.expected_diagnostic,) + assert payload["diagnostics"][0]["message"] == test_case.expected_message + assert all(fragment in text for fragment in test_case.expected_help_fragments), text + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionCase(**asdict(case)) for case in _CASES], + ids=lambda case: case.description, +) +def test_given_described_resource_when_compiling_then_compiles_cleanly( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.described_files + ) + + diagnostics, text = require_sql_analysis_output(result) + assert result.returncode == 0, result.stdout + result.stderr + assert diagnostics == () + assert test_case.expected_message not in text + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionAggregateCase( + description="every required kind undescribed in one project", + files=_ALL_UNDESCRIBED_FILES, + expected_codes=frozenset({"P010"}), + expected_kinds=frozenset(DESCRIPTION_REQUIRED_INPUT_KINDS.values()), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_resource_kind_undescribed_when_compiling_then_one_run_reports_each( + test_case: RequiredDescriptionAggregateCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path, files=test_case.files + ) + + payload: dict[str, Any] = json.loads(result.stdout) + codes: set[str] = {item["code"] for item in payload["diagnostics"]} + kinds: set[str] = {item["message"].split(" ", 1)[0] for item in payload["diagnostics"]} + assert result.returncode == 1, result.stdout + result.stderr + assert codes == test_case.expected_codes + assert kinds == test_case.expected_kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionPlanCase( + description="undescribed model blocks plan", + model_sql="MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + expected_returncode=1, + expected_stderr_fragment=( + "[P010] models/orders.sql:1:1: model 'orders' has no description" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_undescribed_model_when_planning_then_plan_is_blocked( + test_case: RequiredDescriptionPlanCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": REQUIRED_DESCRIPTIONS_PROJECT, + "models/orders.sql": test_case.model_sql, + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "plan") + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_stderr_fragment in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + PathDefaultDescriptionCase( + description="described model sorts before the inheriting model", + model_files=( + ("models/marts/a_model.sql", _OWN_DESCRIPTION_MODEL), + ("models/marts/b_model.sql", _INHERITING_MODEL), + ), + expected_returncode=0, + expected_diagnostics=(), + expected_descriptions=(("a_model", "Own description"), ("b_model", "Mart model")), + ), + PathDefaultDescriptionCase( + description="inheriting model sorts before the described model", + model_files=( + ("models/marts/a_model.sql", _INHERITING_MODEL), + ("models/marts/b_model.sql", _OWN_DESCRIPTION_MODEL), + ), + expected_returncode=0, + expected_diagnostics=(), + expected_descriptions=(("a_model", "Mart model"), ("b_model", "Own description")), + ), + ], + ids=lambda case: case.description, +) +def test_given_path_default_description_when_compiling_then_every_model_keeps_its_description( + test_case: PathDefaultDescriptionCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={"sqlbuild_project.toml": _PATH_DEFAULT_PROJECT, **dict(test_case.model_files)}, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + dag: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "dag", "--json") + ) + + nodes: list[dict[str, Any]] = json.loads(dag.stdout)["nodes"] + assert compiled.returncode == test_case.expected_returncode, compiled.stdout + compiled.stderr + assert tuple(json.loads(compiled.stdout)["diagnostics"]) == test_case.expected_diagnostics + assert dag.returncode == test_case.expected_returncode, dag.stdout + dag.stderr + assert sorted((node["name"], node["description"]) for node in nodes) == list( + test_case.expected_descriptions + ) + + +@pytest.mark.parametrize( + "test_case", + [ + FormatDescriptionResolutionE2ECase( + description="typed-null fixture tests do not hide a path-default description", + extra_files=(), + expected_returncode=0, + expected_faults=(), + ), + FormatDescriptionResolutionE2ECase( + description="a genuinely missing description is still reported", + extra_files=(("models/bare.sql", "MODEL (materialized table);\n\nSELECT 1 AS id\n"),), + expected_returncode=1, + expected_faults=(("models/bare.sql", "description-present"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_inherited_descriptions_when_format_checking_then_faults_match_compile( + test_case: FormatDescriptionResolutionE2ECase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _PATH_DEFAULT_PROJECT, + **dict(_TYPED_NULL_FIXTURE_FILES), + **dict(test_case.extra_files), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "format", "--check", "--json") + ) + + violations: list[dict[str, Any]] = json.loads(result.stdout)["violations"] + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert ( + tuple( + (Path(item["file"]).relative_to(project_dir).as_posix(), item["code"]) + for item in violations + ) + == test_case.expected_faults + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py index a81c41c446..be9fdc3e92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py @@ -37,8 +37,8 @@ def test_given_open_source_when_planning_then_live_columns_reject_missing_refere project_name="source_binding", repo_files={ "sqlbuild_project.toml": 'name = "source_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "sources/orders.yml": "sources:\n - name: raw_orders\n table: raw_orders\n", - "models/orders.sql": 'MODEL (materialized table);\nSELECT missing FROM __source("raw_orders")\n', + "sources/orders.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: raw_orders\n", + "models/orders.sql": 'MODEL (description "Test model orders.", materialized table);\nSELECT missing FROM __source("raw_orders")\n', }, ) database: Path = project / "orders.duckdb" @@ -77,7 +77,7 @@ def test_given_invalid_metadata_when_planning_then_no_warehouse_connection_is_op project_name="metadata_binding", repo_files={ "sqlbuild_project.toml": 'name = "metadata_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "models/orders.sql": "MODEL (materialized table, unique_key [missing]);\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table, unique_key [missing]);\nSELECT 1 AS id\n", }, ) result: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py index 3c267d704f..688cd7f480 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py @@ -22,9 +22,9 @@ description="misspelled MODEL key", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", "models/fct_orders.sql": ( - 'MODEL (\n materialized table,\n descripton "Orders fact.",\n);\n\n' + 'MODEL (description "Test model fct_orders.",\n materialized table,\n descripton "Orders fact.",\n);\n\n' 'SELECT order_id FROM __ref("stg_orders")\n' ), }, @@ -40,7 +40,7 @@ "sqlbuild_project.toml": ( _PROJECT_TOML + '\n[path_defaults.staging]\nmaterialised = "view"\n' ), - "models/staging/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/staging/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\nSELECT 1 AS order_id\n", }, expected_fragments=( "error[D001]", @@ -53,11 +53,11 @@ repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n" " columns:\n - name: order_id\n tpye: INTEGER\n" ), "models/stg_orders.sql": ( - 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model stg_orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' ), }, expected_fragments=( @@ -71,7 +71,7 @@ repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/fct_orders.sql": ( - "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + "MODEL (description 'Test model fct_orders.',\n materialized incremental,\n incremental_strategy append,\n" " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" ), }, @@ -89,7 +89,7 @@ _PROJECT_TOML + '\n[defaults]\nreplay_on_change = "everything"\n' ), "models/fct_orders.sql": ( - "MODEL (materialized incremental, incremental_strategy append);\n\n" + "MODEL (description 'Test model fct_orders.', materialized incremental, incremental_strategy append);\n\n" "SELECT 1 AS order_id\n" ), }, @@ -110,7 +110,7 @@ " - name: raw_orders\n table: orders\n" " primary_keys: order_id\n" ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\nSELECT 1 AS order_id\n", }, expected_fragments=( "error[D006]", @@ -151,7 +151,7 @@ def test_given_unknown_model_config_key_when_compiling_then_fails_naming_key_and " return MaterializationResult(success=True)\n" ), "models/order_snapshot.sql": ( - "MODEL (\n materialized order_snapshot,\n" + "MODEL (description 'Test model order_snapshot.',\n materialized order_snapshot,\n" " config (retention_days 7, partition_column order_id),\n);\n\n" "SELECT 1 AS order_id\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py index ef9a5f99ff..b079f15144 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -44,10 +44,10 @@ 150, 250, ( - "a0fa536e6e64d9be7027ca9a52b4e84431361ec4598aeff1c721fe266fdb096d", - "50e52f18c8bc90479f83d744594cbdefeb13e26a3740ef9bb2ef077f15d2cb5a", - "a08f6c4470d65e2964c10f9c41f6d59ef92089bc5d5bea80104495edb6080d99", - "30809e387b0b096fc592f7da6e429c0c4878c41ed9cc5ea579898fd7e1029687", + "8ef4dcbf6abc8750aee9ae740b09bfd442e6593f5176f059bdc2e1ce01ec538c", + "7c3a736acd69d910a3c612180c982e44a8a07473713f533ab8c5a90a7ecee3e4", + "43a14caf30367ca9e2b966d20ab98b83b2ab8fb36312956f7a11bf001944a0a6", + "b041d70eef587492ab936e98d527fd76895eff69e3d5224e48419e73ed426bf3", ), ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py index 62291c6de3..e5de4a552f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py @@ -271,7 +271,8 @@ def prepare_databricks_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: models_dir: Path = project_dir / "models" / "staging" models_dir.mkdir(parents=True, exist_ok=True) (models_dir / "stg_orders.sql").write_text( - "MODEL (materialized table, unique_key [order_id]);\n\n" + "MODEL (description 'Test model stg_orders.', " + "materialized table, unique_key [order_id]);\n\n" "SELECT * FROM (" "SELECT 1 AS order_id, 1 AS customer_id, 100 AS amount_cents UNION ALL " "SELECT 2 AS order_id, 2 AS customer_id, 200 AS amount_cents" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 9f3b1c19f7..074f14f4a7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -166,14 +166,14 @@ def test_given_python_result_when_running_check_on_databricks_then_persists_node "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -248,13 +248,13 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" database: {catalog_name}\n" f" schema: {schema_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -292,7 +292,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -301,7 +301,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -327,7 +327,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -338,7 +338,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -359,7 +359,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -370,7 +370,7 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -395,7 +395,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -606,7 +606,7 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ DatabricksScenarioLocalReplayE2ETestCase( description="captures databricks fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " date_trunc('DAY', event_ts) AS event_day,\n" @@ -616,7 +616,7 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ "GROUP BY customer_id, date_trunc('DAY', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TIMESTAMP '2026-01-01 08:15:00' " @@ -647,12 +647,12 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ description="reports databricks local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -675,14 +675,14 @@ def test_given_existing_snapshot_targets_when_building_on_databricks_then_apply_ description="reports databricks local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(amount_cents) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -1391,7 +1391,7 @@ def test_given_databricks_invalid_model_when_building_then_underlying_error_is_p project_dir, schema_name = prepare_databricks_waffle_shop(tmp_path=tmp_path) broken_model: Path = project_dir / "models" / "marts" / "databricks_broken_model.sql" broken_model.write_text( - "MODEL (materialized table);\n\nSELECT missing_column FROM (SELECT 1 AS id)", + "MODEL (description 'Test model databricks_broken_model.', materialized table);\n\nSELECT missing_column FROM (SELECT 1 AS id)", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py index 9a2ec73d7f..b1c5dcfaff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_dlt_databricks.py @@ -66,11 +66,12 @@ def test_given_dlt_source_when_loading_to_databricks_then_table_is_materialized_ f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py index e8ab8658ae..e79a4f0c04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -120,11 +120,13 @@ def write_sqlbuild_defer_target_models(*, project_dir: Path) -> None: encoding="utf-8", ) project_dir.joinpath("models", "deferred_upstream.sql").write_text( - "MODEL (materialized table);\n\nSELECT 42 AS order_id\n", + "MODEL (description 'Test model deferred_upstream.', " + "materialized table);\n\nSELECT 42 AS order_id\n", encoding="utf-8", ) project_dir.joinpath("models", "deferred_consumer.sql").write_text( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("deferred_upstream")\n', + 'MODEL (description "Test model deferred_consumer.", ' + 'materialized table);\n\nSELECT order_id FROM __ref("deferred_upstream")\n', encoding="utf-8", ) @@ -233,7 +235,8 @@ def write_sqlbuild_order_status_snapshot(*, project_dir: Path) -> None: encoding="utf-8", ) project_dir.joinpath("models", "order_status_history.sql").write_text( - "MODEL (\n materialized snapshot,\n unique_key [order_id],\n" + "MODEL (description 'Test model order_status_history.'," + "\n materialized snapshot,\n unique_key [order_id],\n" " snapshot_strategy check,\n check_columns [status],\n);\n\n" "SELECT 10 AS order_id, 'open' AS status\n", encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py index 9f48c6bb72..9999bfdeaa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -129,7 +129,7 @@ def test_given_duckdb_dbt_project_when_running_dbt_init_then_generated_project_b (sqlbuild_project_dir / "models").mkdir() (sqlbuild_project_dir / "models" / "local_profile_orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 2 AS order_id\n", + "MODEL (description 'Test model local_profile_orders.', materialized table);\n\nSELECT 2 AS order_id\n", encoding="utf-8", ) @@ -146,7 +146,7 @@ def test_given_duckdb_dbt_project_when_running_dbt_init_then_generated_project_b ) == [(2,)] (sqlbuild_project_dir / "models" / "downstream_orders.sql").write_text( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model downstream_orders.', materialized table);\n\n" 'SELECT order_id FROM __dbt_ref("analytics", "dbt_orders")\n', encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py index 4a799e9533..7d68725aa6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan_safety.py @@ -46,7 +46,8 @@ def test_given_target_model_limit_when_running_dbt_interop_then_refuses_sqlbuild target_settings="\n[targets.main.execution_limits]\nmax_models = 1\n", ) project_dir.joinpath("models", "local_order_ids.sql").write_text( - "MODEL (materialized table);\n\nSELECT 11 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model local_order_ids.', materialized table);\n\nSELECT 11 AS order_id\n", + encoding="utf-8", ) result: subprocess.CompletedProcess[str] = run_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index 504e508064..ef2e0c5869 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -287,6 +287,7 @@ def test_given_debug_command_with_providers_when_running_then_outputs_provider_s class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "debug" """ ).strip() @@ -346,6 +347,7 @@ def test_given_debug_json_with_providers_when_running_then_outputs_provider_summ class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "debug" """ ).strip() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py index b973ae1183..9d54225c5c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/diff/helpers.py @@ -50,6 +50,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT * FROM (VALUES (1, 1, TIMESTAMP '2026-04-01 09:15:00', 'placed', 100), @@ -61,7 +62,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/staging/stg_orders.sql": dedent( """ - MODEL (materialized view); + MODEL (description "Test model stg_orders.", materialized view); SELECT order_id, @@ -75,7 +76,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/orders_snapshot.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_snapshot.", materialized table, unique_key [order_id], cursor ordered_at, @@ -102,7 +103,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/customer_totals.sql": dedent( """ - MODEL ( + MODEL (description "Test model customer_totals.", materialized table, unique_key [customer_id] ); @@ -118,7 +119,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/intermediate/orders_sparse.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_sparse.", materialized table, unique_key [order_id] ); @@ -133,7 +134,7 @@ def prepare_diff_project(tmp_path: Path) -> Path: + "\n", "models/marts/daily_revenue.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model daily_revenue.", materialized table); SELECT CAST(ordered_at AS DATE) AS revenue_date, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py index 876eacafb8..93b6ce36ab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/helpers.py @@ -30,7 +30,8 @@ def write_format_performance_project( tests_dir.mkdir(parents=True) for model_index in range(model_count): models_dir.joinpath(f"orders_{model_index:05d}.sql").write_text( - "MODEL (columns (order_id (type INTEGER), note (type VARCHAR)));\n" + 'MODEL (description "Test model.", ' + "columns (order_id (type INTEGER), note (type VARCHAR)));\n" f"SELECT {model_index} AS order_id, 'ready' AS note\n", encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py index ea215f1c40..08cbb50548 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py @@ -21,7 +21,7 @@ [ RuleFixTestCase( "verification failure", - "MODEL (sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item", + 'MODEL (description "Test model.", sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item', "refused", "SQBRSQL005", 1, @@ -30,7 +30,7 @@ ), RuleFixTestCase( "generated dependency region", - "MODEL ();\n" + 'MODEL (description "Test model.");\n' 'WITH unused AS (SELECT customer_id FROM __ref("customers")) SELECT 2 AS item', "refused", "SQBRSQL005", @@ -40,7 +40,7 @@ ), RuleFixTestCase( "fixed point with overlapping unused CTE deletions", - "MODEL ();\n" + 'MODEL (description "Test model.");\n' "WITH first AS (SELECT 1 AS item), second AS (SELECT item FROM first), " "third AS (SELECT item FROM second) SELECT 2 AS item", "applied", @@ -59,7 +59,9 @@ def test_given_rule_fixes_when_running_cli_then_reports_verified_or_refused_chan model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text(test_case.sql) - (model.parent / "customers.sql").write_text("MODEL (); SELECT 1 AS customer_id") + (model.parent / "customers.sql").write_text( + 'MODEL (description "Test model customers."); SELECT 1 AS customer_id' + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -104,7 +106,9 @@ def test_given_wide_or_deep_unused_ctes_when_fixing_then_work_is_bounded( ctes: list[str] = [f"items_0 AS (SELECT {columns})"] for index in range(1, test_case.depth): ctes.append(f"items_{index} AS (SELECT * FROM items_{index - 1})") - model.write_text("MODEL (); WITH " + ", ".join(ctes) + " SELECT 1 AS order_id") + model.write_text( + 'MODEL (description "Test model."); WITH ' + ", ".join(ctes) + " SELECT 1 AS order_id" + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py index 3283901d5c..66db7920be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py @@ -26,7 +26,8 @@ def prepare_freshness_project( include_managed_source=include_managed_source, ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), } repo_files.update( @@ -37,7 +38,7 @@ def prepare_freshness_project( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_managed(ctx):\n" - " return [{'event_id': 5}]\n" + " '''Test loader raw_managed.'''\n return [{'event_id': 5}]\n" ) }, }[include_managed_source] @@ -62,10 +63,12 @@ def prepare_multi_schema_freshness_project(*, tmp_path: Path) -> Path: ), "sources/raw.yml": freshness_sources_yml(include_error_source=False), "models/dev_orders.sql": ( - 'MODEL (materialized table, schema dev);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model dev_orders.", ' + 'materialized table, schema dev);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/mart_orders.sql": ( - 'MODEL (materialized table, schema mart);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model mart_orders.", ' + 'materialized table, schema mart);\n\nSELECT * FROM __source("raw_orders")\n' ), }, ) @@ -88,6 +91,7 @@ def freshness_sources_yml( False: "", True: ( " - name: raw_error\n" + " description: Test source raw_error.\n" " expression: SELECT 3 AS order_id\n" " freshness:\n" " strategy: sql\n" @@ -99,6 +103,7 @@ def freshness_sources_yml( False: "", True: ( " - name: raw_managed\n" + " description: Test source raw_managed.\n" " managed: true\n" " expression: SELECT 5 AS event_id\n" " freshness:\n" @@ -109,12 +114,13 @@ def freshness_sources_yml( }[include_managed_source] return ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" f"{freshness}" f"{error_source}" f"{managed_source}" " - name: raw_unknown\n" + " description: Test source raw_unknown.\n" " expression: SELECT 2 AS order_id\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py index a5b67872a8..4706f2ae5f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py @@ -70,7 +70,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir description="initialized project builds Python nodes and helpers under python", project_files={ "models/marts/fact_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\nSELECT 1 AS order_id\n" ), "python/_helpers.py": "def order_count(rows):\n return len(rows)\n", "python/tasks/orders.py": ( @@ -78,7 +78,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir "from sqlbuild.tasks import task\n\n\n" "@task\n" "def count_orders(ctx):\n" - " return ctx.result(payload={'order_count': order_count([1, 2])})\n" + " '''Test task count_orders.'''\n return ctx.result(payload={'order_count': order_count([1, 2])})\n" ), "python/assets/orders_export.py": ( "from python.tasks.orders import count_orders\n" @@ -86,7 +86,7 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir "from sqlbuild.refs import model\n\n\n" "@asset(depends_on=(model('fact_orders'), count_orders))\n" "def orders_export(ctx):\n" - " return ctx.result(payload={'ready': True})\n" + " '''Test asset orders_export.'''\n return ctx.result(payload={'ready': True})\n" ), }, unexpected_paths=("tasks", "assets", "checks", "loaders", "factories", "libs"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 68823f71ca..e50ef55753 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -59,7 +59,7 @@ def prepare_janitor_project( "sqlbuild_project.toml": project_config, "models/orders.sql": dedent( """ - MODEL (); + MODEL (description "Test model orders."); SELECT 1 AS order_id """ @@ -285,7 +285,8 @@ def prepare_archive_janitor_project( 'materialized = "table"\n' ), **{ - f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + f"models/{model_name}.sql": f"MODEL (description 'Test model.');\n\n" + f"SELECT 1 AS {model_name}_id\n" for model_name in model_names }, f"adapters/{AGED_JANITOR_ADAPTER_NAME}.py": AGED_JANITOR_ADAPTER_SOURCE, @@ -393,7 +394,8 @@ def prepare_two_target_janitor_project( 'materialized = "table"\n' ), **{ - f"models/{model_name}.sql": f"MODEL ();\n\nSELECT 1 AS {model_name}_id\n" + f"models/{model_name}.sql": f"MODEL (description 'Test model.');\n\n" + f"SELECT 1 AS {model_name}_id\n" for model_name in model_names }, }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 1be089a3c1..810db559b5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -202,12 +202,13 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ """ ).strip() + "\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", - "models/safe_orders.sql": "MODEL (schema safe,);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", + "models/safe_orders.sql": "MODEL (description 'Test model safe_orders.', schema safe,);\n\nSELECT 1 AS order_id\n", "sources/raw.yml": dedent( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index 8aa3fa0063..dc0169530d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -128,7 +128,8 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i assert origin_build.returncode == 0, origin_build.stdout + origin_build.stderr (project_dir / "models" / "revenue.sql").unlink() (project_dir / "models" / "daily_revenue.sql").write_text( - "MODEL (\n migrate_from revenue,\n);\n\nSELECT 1 AS revenue_id\n", encoding="utf-8" + "MODEL (description 'Test model daily_revenue.',\n migrate_from revenue,\n);\n\nSELECT 1 AS revenue_id\n", + encoding="utf-8", ) rename_build: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", "--target", "prod"), project_dir=project_dir diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py index 3ec718855d..0603f89712 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/helpers.py @@ -13,13 +13,16 @@ def prepare_lineage_cache_project(*, tmp_path: Path) -> Path: repo_files={ "sqlbuild_project.toml": 'name = "lineage_cache_project"\nadapter = "duckdb"\n', "models/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS order_id, 10 AS customer_id\n" + "MODEL (description 'Test model stg_orders.', " + "materialized view);\n\nSELECT 1 AS order_id, 10 AS customer_id\n" ), "models/stg_customers.sql": ( - "MODEL (materialized view);\n\nSELECT 10 AS customer_id\n" + "MODEL (description 'Test model stg_customers.', " + "materialized view);\n\nSELECT 10 AS customer_id\n" ), "models/fact_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model fact_orders.", ' + 'materialized view);\n\nSELECT * FROM __ref("stg_orders")\n' ), }, ) @@ -49,16 +52,18 @@ def prepare_diamond_lineage_project(*, tmp_path: Path) -> Path: files: dict[str, str] = { "sqlbuild_project.toml": 'name = "diamond_lineage"\nadapter = "duckdb"\n', - "models/orders_0.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id, 10 AS amount\n", + "models/orders_0.sql": "MODEL (description 'Test model orders_0.', " + "materialized view);\n\nSELECT 1 AS order_id, 10 AS amount\n", } for layer in range(1, DIAMOND_LAYERS + 1): previous: str = f"orders_{layer - 1}" for side in ("left", "right"): files[f"models/orders_{side}_{layer}.sql"] = ( - f'MODEL (materialized view);\n\nSELECT order_id, amount FROM __ref("{previous}")\n' + 'MODEL (description "Test model.", ' + f'materialized view);\n\nSELECT order_id, amount FROM __ref("{previous}")\n' ) files[f"models/orders_{layer}.sql"] = ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model.', materialized view);\n\n" "SELECT l.order_id, l.amount + r.amount AS amount\n" f'FROM __ref("orders_left_{layer}") AS l\n' f'JOIN __ref("orders_right_{layer}") AS r ON l.order_id = r.order_id\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py index e42776e88d..27d9a44aab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py @@ -105,7 +105,7 @@ def test_given_directional_selectors_when_expanding_then_depth_and_exclusions_ar for name in ("stg_orders.sql", "stg_customers.sql"): (project_dir / "models" / name).rename(staging_dir / name) (project_dir / "models" / "fact_orders.sql").write_text( - 'MODEL (materialized view, tags [finance]);\nSELECT * FROM __ref("stg_orders")\n', + 'MODEL (description "Test model fact_orders.", materialized view, tags [finance]);\nSELECT * FROM __ref("stg_orders")\n', encoding="utf-8", ) @@ -336,7 +336,7 @@ def test_given_authored_dependency_change_when_lineage_runs_then_rebuilds_struct ) assert first.returncode == 0, first.stdout + first.stderr (project_dir / "models/fact_orders.sql").write_text( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model fact_orders.', materialized view);\n\n" 'SELECT orders.order_id FROM __ref("stg_orders") AS orders\n' 'JOIN __ref("stg_customers") AS customers USING (customer_id)\n', encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py index 37704c6616..a35a991aea 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py @@ -30,7 +30,10 @@ def assert_source_only_ingress_dependency_case( project_name="source_loader_schema_behavior", repo_files={ **build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n" "from python.tasks.prepare import prepare_events\n\n" @@ -39,13 +42,13 @@ def assert_source_only_ingress_dependency_case( " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [\n" + " '''Test loader fetch_events.'''\n return [\n" " {'event_id': 1, 'load_seq': 1},\n" " {'event_id': 2, 'load_seq': 1},\n" " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), @@ -55,6 +58,7 @@ def assert_source_only_ingress_dependency_case( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" + " '''Test task prepare_events.'''\n" " Path(__file__).parents[2].joinpath('prepared.txt').write_text('prepared')\n" " return ctx.result()\n" ), @@ -233,7 +237,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -249,7 +253,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " type: INTEGER\n" " - name: load_seq\n" " type: INTEGER\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -270,7 +274,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_order_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_order_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -301,7 +305,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_customers(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_customers.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'customer_id': 1, 'plan_name': 'basic', 'load_seq': 1},\n" " {'customer_id': 2, 'plan_name': 'plus', 'load_seq': 1},\n" @@ -316,6 +320,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_prices(ctx):\n" + " '''Test loader fetch_prices.'''\n" " classic_price = 600 if ctx.current_cursor_value is None else 650\n" " return [\n" " {'waffle_type': 'classic', 'price_cents': classic_price, 'load_seq': 1},\n" @@ -323,7 +328,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n\n" "@loader(depends_on=[fetch_order_events, fetch_prices])\n" "def raw_orders(ctx):\n" - " events = ctx.loader(fetch_order_events)\n" + " '''Test loader raw_orders.'''\n events = ctx.loader(fetch_order_events)\n" " prices = ctx.loader(fetch_prices)\n" " cursor = ctx.query(\n" " f'SELECT e.order_id, e.customer_id, e.waffle_type, e.quantity, '\n" @@ -344,7 +349,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n\n" "@loader(depends_on=[fetch_customers])\n" "def raw_customers(ctx):\n" - " customers = ctx.loader(fetch_customers)\n" + " '''Test loader raw_customers.'''\n customers = ctx.loader(fetch_customers)\n" " cursor = ctx.query(\n" " f'SELECT customer_id, plan_name, load_seq FROM {customers.destination} '\n" " f'ORDER BY customer_id'\n" @@ -355,7 +360,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " ]\n" ), "models/fact_waffle_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_waffle_orders.', materialized table);\n\n" "SELECT\n" " o.order_id,\n" " o.customer_id,\n" @@ -369,7 +374,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - 'LEFT JOIN __source("raw_customers") c ON o.customer_id = c.customer_id\n' ), "models/customer_revenue.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_revenue.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " COALESCE(plan_name, 'unknown') AS plan_name,\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py index dca6faccec..21adbee791 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py @@ -58,11 +58,12 @@ def test_given_dlt_sql_database_source_when_loading_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, @@ -132,6 +133,7 @@ def test_given_dlt_filesystem_source_when_loading_then_table_is_materialized_and ' file_glob: "orders.jsonl"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " reader: jsonl\n" " write_disposition: replace\n" ), @@ -141,7 +143,7 @@ def test_given_dlt_filesystem_source_when_loading_then_table_is_materialized_and '{"order_id": 3, "amount": 30}\n' ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, @@ -210,13 +212,14 @@ def test_given_dlt_rest_api_source_when_loading_then_table_is_materialized_and_q f' base_url: "{base_url}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " endpoint:\n" ' path: "orders"\n' ' data_selector: "$"\n' " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 255c678e4a..40ffb4dc45 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -133,7 +133,7 @@ def test_given_non_contract_loader_when_late_column_appears_then_existing_target repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -147,7 +147,7 @@ def test_given_non_contract_loader_when_late_column_appears_then_existing_target "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -309,10 +309,10 @@ def test_given_two_terminal_sources_when_sharing_intermediate_then_intermediate_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" - " - name: raw_order_metrics\n" + " - name: raw_order_metrics\n description: Test source raw_order_metrics.\n" " managed: true\n" " write_strategy: table\n" ), @@ -323,21 +323,21 @@ def test_given_two_terminal_sources_when_sharing_intermediate_then_intermediate_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_orders(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_orders.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" " return [{'event_id': next_seq, 'load_seq': next_seq}]\n\n" "@loader(depends_on=[fetch_orders])\n" "def raw_orders(ctx):\n" - " orders = ctx.loader(fetch_orders)\n" + " '''Test loader raw_orders.'''\n orders = ctx.loader(fetch_orders)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {orders.destination} ORDER BY event_id'\n" " )\n" " return [{'event_id': row[0]} for row in cursor.fetchall()]\n\n" "@loader(depends_on=[fetch_orders])\n" "def raw_order_metrics(ctx):\n" - " orders = ctx.loader(fetch_orders)\n" + " '''Test loader raw_order_metrics.'''\n orders = ctx.loader(fetch_orders)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {orders.destination} ORDER BY event_id'\n" " )\n" @@ -392,7 +392,7 @@ def test_given_chained_loader_project_when_loading_source_then_runs_dependencies repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -407,14 +407,14 @@ def test_given_chained_loader_project_when_loading_source_then_runs_dependencies " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " source_cursor = ctx.source('raw_events').max('event_id')\n" + " '''Test loader fetch_events.'''\n source_cursor = ctx.source('raw_events').max('event_id')\n" " return [\n" " {'event_id': 1, 'load_seq': 1},\n" " {'event_id': 2, 'load_seq': 1},\n" " ] if source_cursor is None else []\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status, {events.current_cursor_value} AS max_seq " 'FROM {events.destination}")\n' @@ -464,7 +464,7 @@ def test_given_chained_loader_project_when_building_source_model_then_runs_loade repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -478,17 +478,17 @@ def test_given_chained_loader_project_when_building_source_model_then_runs_loade " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status FROM {events.destination}\")\n" ), ) | { "models/fact_events.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_events.', materialized table);\n\n" 'SELECT event_id, status FROM __source("raw_events")\n' ) }, @@ -543,7 +543,7 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " columns:\n" " - name: event_id\n" @@ -557,17 +557,17 @@ def test_given_chained_loader_project_when_running_source_model_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, 'loaded' AS status FROM {events.destination}\")\n" ), ) | { "models/fact_events.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_events.', materialized table);\n\n" 'SELECT event_id, status FROM __source("raw_events")\n' ) }, @@ -631,7 +631,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run **build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -643,7 +643,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run f"from python.{test_case.dependency_dir}.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" - " payload = ctx.result_of(node_function=prepare_events).payload\n" + " '''Test loader raw_events.'''\n payload = ctx.result_of(node_function=prepare_events).payload\n" " return [{'event_id': payload['event_id']}]\n" ), ), @@ -752,9 +752,9 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" - " - name: raw_rollup\n" + " - name: raw_rollup\n description: Test source raw_rollup.\n" " managed: true\n" ), loader_py=( @@ -763,15 +763,15 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'price_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_prices(ctx):\n" - " return [{'price_id': 1}]\n\n" + " '''Test loader fetch_prices.'''\n return [{'price_id': 1}]\n\n" "@loader(depends_on=[fetch_events, fetch_prices])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " prices = ctx.loader(fetch_prices)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination} UNION ALL " @@ -780,16 +780,16 @@ def test_given_source_only_selection_when_skipping_complex_intermediates_then_va " {'name': 'page_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_pages(ctx):\n" - " return [{'page_id': 1}]\n\n" + " '''Test loader fetch_pages.'''\n return [{'page_id': 1}]\n\n" "@loader(depends_on=[fetch_pages], write_strategy='table', " "columns=[{'name': 'page_id', 'type': 'INTEGER'}])\n" "def normalize_pages(ctx):\n" - " pages = ctx.loader(fetch_pages)\n" + " '''Test loader normalize_pages.'''\n pages = ctx.loader(fetch_pages)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT page_id FROM {pages.destination}')\n\n" "@loader(depends_on=[normalize_pages])\n" "def raw_rollup(ctx):\n" - " pages = ctx.loader(normalize_pages)\n" + " '''Test loader raw_rollup.'''\n pages = ctx.loader(normalize_pages)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT page_id FROM {pages.destination}')\n" ), @@ -879,7 +879,9 @@ def test_given_chained_loader_project_when_selecting_loader_then_expands_expecte tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" @@ -887,10 +889,10 @@ def test_given_chained_loader_project_when_selecting_loader_then_expands_expecte " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'load_seq': 1}, {'event_id': 2, 'load_seq': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'load_seq': 1}, {'event_id': 2, 'load_seq': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -949,17 +951,19 @@ def test_given_chained_loader_project_when_excluding_dependency_then_prunes_depe tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -1000,17 +1004,19 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r tmp_path=tmp_path, project_name="source_loader_schema_behavior", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(destination='custom_fetch_events', write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' "SELECT event_id, '{events.table_name}' AS source_table " 'FROM {events.destination}")\n' @@ -1049,10 +1055,10 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'amount', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'amount': 100}, {'event_id': 2, 'amount': 200}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'amount': 100}, {'event_id': 2, 'amount': 200}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1077,7 +1083,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1088,7 +1094,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1109,7 +1115,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is not None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is not None:\n" " return [\n" " {'event_id': 2, 'amount': 250, 'load_seq': 1},\n" " {'event_id': 3, 'amount': 300, 'load_seq': 1},\n" @@ -1120,7 +1126,7 @@ def test_given_chained_loader_project_when_intermediate_has_custom_target_then_r " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1144,7 +1150,7 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1210,9 +1216,9 @@ def test_given_loader_dependency_failure_when_loading_then_only_dependents_skip( repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1222,15 +1228,15 @@ def test_given_loader_dependency_failure_when_loading_then_only_dependents_skip( " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ), ) @@ -1275,9 +1281,9 @@ def test_given_loader_dependency_failure_when_building_then_only_dependents_skip repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1287,23 +1293,23 @@ def test_given_loader_dependency_failure_when_building_then_only_dependents_skip " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ) | { "models/bad_model.sql": ( - 'MODEL (materialized table);\n\nSELECT event_id FROM __source("raw_bad")\n' + 'MODEL (description "Test model bad_model.", materialized table);\n\nSELECT event_id FROM __source("raw_bad")\n' ), "models/ok_model.sql": ( - 'MODEL (materialized table);\n\nSELECT event_id FROM __source("raw_ok")\n' + 'MODEL (description "Test model ok_model.", materialized table);\n\nSELECT event_id FROM __source("raw_ok")\n' ), }, ) @@ -1350,9 +1356,9 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_bad\n" + " - name: raw_bad\n description: Test source raw_bad.\n" " managed: true\n" - " - name: raw_ok\n" + " - name: raw_ok\n description: Test source raw_ok.\n" " managed: true\n" " write_strategy: table\n" ), @@ -1362,15 +1368,15 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_bad(ctx):\n" - " raise RuntimeError('boom')\n\n" + " '''Test loader fetch_bad.'''\n raise RuntimeError('boom')\n\n" "@loader(depends_on=[fetch_bad])\n" "def raw_bad(ctx):\n" - " events = ctx.loader(fetch_bad)\n" + " '''Test loader raw_bad.'''\n events = ctx.loader(fetch_bad)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n\n' "@loader\n" "def raw_ok(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_ok.'''\n return [{'event_id': 1}]\n" ), ), ) @@ -1407,7 +1413,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_contract_events\n" + " - name: raw_contract_events\n description: Test source raw_contract_events.\n" " managed: true\n" " write_strategy: table\n" " contract: enforced\n" @@ -1419,7 +1425,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_contract_events(ctx):\n" - " return [{'event_id': 1, 'extra_note': 'not declared'}]\n" + " '''Test loader raw_contract_events.'''\n return [{'event_id': 1, 'extra_note': 'not declared'}]\n" ), ), ), @@ -1430,7 +1436,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_contract_events\n" + " - name: raw_contract_events\n description: Test source raw_contract_events.\n" " managed: true\n" " write_strategy: table\n" " contract: enforced\n" @@ -1444,7 +1450,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_contract_events(ctx):\n" - " return [{'event_id': 1}]\n" + " '''Test loader raw_contract_events.'''\n return [{'event_id': 1}]\n" ), ), ), @@ -1455,7 +1461,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1469,7 +1475,7 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1, 'amount': 100}]\n" " return [{'event_id': 2, 'load_seq': 2, 'amount': 'one hundred'}]\n" ), @@ -1480,17 +1486,19 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ command=("--no-color", "load", "--select", "+raw_events"), expected_error_fragment="contract has extra columns: extra_note", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', contract='enforced', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1, 'extra_note': 'not declared'}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1, 'extra_note': 'not declared'}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT event_id FROM {events.destination}")\n' ), @@ -1501,15 +1509,17 @@ def test_given_loader_dependency_failure_when_loading_json_then_reports_skipped_ command=("--no-color", "load", "--select", "+raw_events"), expected_error_fragment="returned no rows and has no destination declared", repo_files=build_schema_behavior_project_files( - source_yaml=("sources:\n - name: raw_events\n managed: true\n"), + source_yaml=( + "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n" + ), loader_py=( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def fetch_events(ctx):\n" - " ctx.execute_sql('SELECT 1')\n\n" + " '''Test loader fetch_events.'''\n ctx.execute_sql('SELECT 1')\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS ' 'SELECT * FROM {events.destination}")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py index 7e0da24b9e..1cd132172c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load_relation_guard.py @@ -16,17 +16,17 @@ _SOURCES: str = ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" ) _LOADERS: str = ( "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" "@loader(depends_on=[raw_regions])\n" "def raw_customers(ctx):\n" - " table = 'raw_' + 'regions'\n" + " '''Test loader raw_customers.'''\n table = 'raw_' + 'regions'\n" " ctx.query(f'SELECT count(*) FROM {table}')\n" " return [{'id': 1}]\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 26b32b58a4..d916c9b809 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -20,7 +20,10 @@ f'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{DATABASE_FILE}"\n' "\n[migrations]\nold_name_views = false\n" ) -_SOURCES_YML: str = "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" +_SOURCES_YML: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" +) def orders_sql(*, migrate_from: str) -> str: @@ -28,7 +31,7 @@ def orders_sql(*, migrate_from: str) -> str: migration: str = {"": ""}.get(migrate_from, f' migrate_from "{migrate_from}",\n') return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -136,7 +139,7 @@ def fct_orders_sql(*, columns: str, extra_config: str = "", replay: bool = True) """Return an incremental orders fact model projecting the given value columns.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -293,7 +296,7 @@ def old_name_model_sql( """Return a model of the given materialization over raw orders.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" f"{_OLD_NAME_HEADERS[materialized]}" f"{extra_config}" ");\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py index 4622bb60a0..fad925b216 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/model_migrations/test_old_name_views.py @@ -44,7 +44,7 @@ "from sqlbuild.tasks import task\n\n\n" "@task\n" "def report(ctx):\n" - f' ctx.query("SELECT count(*) FROM {OLD_NAME_SCHEMA}.{OLD_NAME_ORIGIN}").fetchall()\n' + f' """Test task report."""\n ctx.query("SELECT count(*) FROM {OLD_NAME_SCHEMA}.{OLD_NAME_ORIGIN}").fetchall()\n' ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py index 66e3ac3042..d41d8bb80a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/motherduck/helpers.py @@ -45,7 +45,7 @@ def prepare_motherduck_build_project(*, tmp_path: Path) -> tuple[Path, str]: schema_name=schema_name, ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "SELECT 1 AS order_id, 'classic' AS waffle_name UNION ALL " "SELECT 2 AS order_id, 'liege' AS waffle_name" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 4a7dc4a4a7..6dd49631fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -18,14 +18,15 @@ f'[connection]\ndatabase = "{DIAMOND_DATABASE_FILE}"\n' ) _DIAMOND_SOURCES_YML: str = ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ) _DIAMOND_ROOT_PATH: str = "models/orders_0.sql" def _incremental_header(*, extra_config: str) -> str: return ( - "MODEL (\n" + "MODEL (description 'Test model orders_0.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -107,6 +108,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: _MISSING_SOURCE_TABLES_SOURCES_YML: str = """sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders freshness: @@ -114,6 +116,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: column: updated_at type: timestamp - name: raw_customers + description: Test source raw_customers. schema: raw table: customers freshness: @@ -121,6 +124,7 @@ def prepare_changed_incremental_diamond_project(*, tmp_path: Path) -> Path: column: updated_at type: timestamp - name: raw_payments + description: Test source raw_payments. schema: raw table: payments freshness: @@ -145,13 +149,16 @@ def prepare_missing_source_tables_project(*, tmp_path: Path) -> Path: ), "sources/raw.yml": _MISSING_SOURCE_TABLES_SOURCES_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), "models/customers.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_customers")\n' + 'MODEL (description "Test model customers.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_customers")\n' ), "models/payments.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_payments")\n' + 'MODEL (description "Test model payments.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_payments")\n' ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py index 21e1406eb4..d27385997e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_missing_source_tables.py @@ -31,7 +31,7 @@ "error[S405]", "1 source table read by selected resources does not exist in the warehouse", "source 'raw_payments' (raw.payments), read by payments", - "'raw_payments' at sources/raw.yml:16", + "'raw_payments' at sources/raw.yml:18", ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py index 5a90a99e36..412e4bfbb8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py @@ -30,14 +30,14 @@ def test_given_physical_source_shape_when_rebinding_then_rejects_downstream_befo ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n table: raw_orders\n schema: main\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: raw_orders\n schema: main\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized table, schema analytics); SELECT * FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table, schema analytics); SELECT * FROM __source("raw_orders")' ) (tmp_path / "models/report.sql").write_text( - 'MODEL (materialized table, schema analytics); SELECT missing FROM __ref("orders")' + 'MODEL (description "Test model report.", materialized table, schema analytics); SELECT missing FROM __ref("orders")' ) result: subprocess.CompletedProcess[str] = run_sqb( command=test_case.command, project_dir=tmp_path diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py index 783b12cb8b..c0b04aa5ca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py @@ -95,7 +95,7 @@ def test_given_removed_interface_when_running_cli_then_reports_unknown_argument( ), RemovedConfigTestCase( "removed unchanged MODEL field", - "MODEL (materialized table, run_despite_unchanged always); SELECT 1 AS order_id", + "MODEL (description 'Test model.', materialized table, run_despite_unchanged always); SELECT 1 AS order_id", "run_despite_unchanged", "models/orders.sql", "", @@ -158,7 +158,7 @@ def test_given_removed_config_when_running_cli_then_explains_direct_mode( ) -> None: files: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (materialized view);\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\nSELECT 1 AS order_id", } files[test_case.filename] = test_case.prefix + test_case.content project: Path = prepare_inline_project( @@ -193,7 +193,7 @@ def test_given_prepare_version_hook_when_running_cli_then_rejects_removed_hook( project_name="removed_hook", repo_files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', - "models/orders.sql": "MODEL (materialized custom_order);\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized custom_order);\nSELECT 1 AS order_id", "materializations/custom_order.py": ( "from sqlbuild.executor.custom.models import PrepareVersionContext\n\n" "def prepare_version(ctx: PrepareVersionContext):\n pass\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index 56c6e93818..b37f33056d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -471,7 +471,7 @@ def orders_sql(*, migrate_from: str = "", where: str = "") -> str: migrate_from, f' migrate_from "{migrate_from}",\n migrate_force true,\n' ) return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -488,7 +488,10 @@ def orders_sql(*, migrate_from: str = "", where: str = "") -> str: def report_view_sql() -> str: """Return a project view that reads the destination by name.""" - return f'MODEL (materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + return ( + 'MODEL (description "Test model.", ' + f'materialized view);\n\nSELECT order_id FROM __ref("{DESTINATION_MODEL}")\n' + ) def write_migration_project( @@ -507,7 +510,8 @@ def write_migration_project( ) + _JANITOR_CONFIG, "sources/raw.yml": ( - f"sources:\n - name: raw_orders\n schema: {raw_schema_name}\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + f" schema: {raw_schema_name}\n" " table: raw_orders\n" ), } @@ -636,7 +640,7 @@ def fact_orders_sql(*, amount: str = "amount_cents", columns: str = "") -> str: declarations: str = {"": ""}.get(columns, f" columns ({columns}),\n") return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -717,7 +721,7 @@ def old_name_model_sql( migration: str = {"": ""}.get(migrate_from, f" migrate_from {migrate_from},\n") return ( - f"MODEL (\n{_OLD_NAME_HEADERS[materialized]}{migration});\n\n" + f"MODEL (description 'Test model.',\n{_OLD_NAME_HEADERS[materialized]}{migration});\n\n" f'SELECT {columns} FROM __source("raw_orders")\n' ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py index 782d927e5c..f02490f61e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_dlt_postgres.py @@ -68,11 +68,12 @@ def test_given_dlt_source_when_loading_to_postgres_then_table_is_materialized_an f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 1284767841..8cdd464dc8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -262,14 +262,14 @@ def test_given_python_result_when_running_check_on_postgres_then_persists_node_r "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -390,7 +390,7 @@ def test_given_source_deferral_env_when_building_on_postgres_then_reads_prod_and ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -403,10 +403,10 @@ def test_given_source_deferral_env_when_building_on_postgres_then_reads_prod_and "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -485,7 +485,7 @@ def test_given_partial_source_type_enforcement_when_building_on_postgres_then_ca ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {schema_name}\n" " type_enforcement: true\n" " columns:\n" @@ -493,7 +493,7 @@ def test_given_partial_source_type_enforcement_when_building_on_postgres_then_ca " type: INTEGER\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT order_id, status, amount_cents FROM __source("raw_orders")' ), }, @@ -709,7 +709,7 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write PostgresScenarioLocalReplayE2ETestCase( description="captures postgres fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " DATE_TRUNC('day', event_ts) AS event_day,\n" @@ -720,7 +720,7 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write "GROUP BY customer_id, DATE_TRUNC('day', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, CAST('2026-01-01 08:15:00' AS TIMESTAMP)" @@ -751,12 +751,12 @@ def test_given_loader_strategy_project_when_loading_twice_on_postgres_then_write description="reports postgres local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -855,7 +855,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -870,10 +870,10 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -933,7 +933,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -942,7 +942,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -968,7 +968,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -979,7 +979,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1000,7 +1000,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1011,7 +1011,7 @@ def test_given_chained_loader_project_when_loading_on_postgres_then_runs_loader_ " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1037,7 +1037,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_postgres_then repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index b2242b36e8..cca5fb17fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -63,6 +63,7 @@ def test_given_provider_exporter_when_building_then_receives_redacted_events_bef from sqlbuild.providers import Provider class EventSink(Provider): + '''Test provider EventSink.''' path: Path | None = None def setup(self, ctx): @@ -106,7 +107,7 @@ def failing_exporter(event: LifecycleEvent, event_sink: EventSink) -> None: + "\n", "models/orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model orders.", materialized table); SELECT 'do-not-export-this-sql' AS secret_value """ @@ -191,7 +192,7 @@ def export_output(record: CommandOutputRecord) -> None: """ ).strip() + "\n", - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, ) @@ -237,7 +238,7 @@ def test_given_configured_provider_and_sink_when_lineage_runs_then_skips_extensi "from pathlib import Path\n" "from sqlbuild.providers import Provider\n" "class OutputProvider(Provider):\n" - " def setup(self, ctx):\n" + " '''Test provider OutputProvider.'''\n def setup(self, ctx):\n" " del ctx\n" f" Path({str(provider_marker)!r}).write_text('initialized', encoding='utf-8')\n" " def write(self, record):\n" @@ -252,7 +253,7 @@ def test_given_configured_provider_and_sink_when_lineage_runs_then_skips_extensi "def export_output(record, output_provider: OutputProvider):\n" " output_provider.write(record)\n" ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, ) @@ -303,7 +304,7 @@ def test_given_no_exporters_and_exploding_provider_when_non_provider_command_run f"Path({str(helper_marker_path)!r}).write_text('imported', encoding='utf-8')\n" "def encode(value):\n return value\n" ), - "models/example.sql": "MODEL (materialized view);\n\nSELECT 1 AS value\n", + "models/example.sql": "MODEL (description 'Test model example.', materialized view);\n\nSELECT 1 AS value\n", }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 0ec54603f1..1836ed8fa5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -53,6 +53,7 @@ class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' marker_path: str = Field(validation_alias="MARKER_PATH") def setup(self, ctx): @@ -124,6 +125,7 @@ def test_given_provider_usages_when_planning_then_text_and_json_include_selected class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' label: str = "plan-output" """ ).strip() @@ -136,6 +138,7 @@ class MarkerProvider(Provider): @loader def raw_orders(marker_provider: MarkerProvider): + '''Test loader raw_orders.''' return [{"order_id": 1, "amount": 10}] """ ).strip() @@ -148,6 +151,7 @@ def raw_orders(marker_provider: MarkerProvider): @task def publish_orders(marker_provider: MarkerProvider): + '''Test task publish_orders.''' marker_provider.label """ ).strip() @@ -160,6 +164,7 @@ def publish_orders(marker_provider: MarkerProvider): @hook def mark_pre(marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.label """ ).strip() @@ -187,6 +192,7 @@ def materialize( """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -199,7 +205,7 @@ def materialize( + "\n", "models/stg_orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model stg_orders.", materialized copy_table, pre_hooks [python("mark_pre")] ); @@ -300,6 +306,7 @@ def test_given_project_with_provider_when_running_commands_then_provider_is_inje class MarkerProvider(Provider): + '''Test provider MarkerProvider.''' marker_path: str = Field(validation_alias="MARKER_PATH") label: str = "provider-e2e" @@ -324,6 +331,7 @@ def mark(self, value): @task def provider_task(ctx, marker_provider: MarkerProvider): + '''Test task provider_task.''' marker_provider.mark(f"run_task:{marker_provider.label}") """ ).strip() @@ -336,6 +344,7 @@ def provider_task(ctx, marker_provider: MarkerProvider): @asset def provider_asset(ctx, marker_provider: MarkerProvider): + '''Test asset provider_asset.''' marker_provider.mark(f"build_asset:{marker_provider.label}") """ ).strip() @@ -348,6 +357,7 @@ def provider_asset(ctx, marker_provider: MarkerProvider): @check(depends_on=()) def provider_check(ctx, marker_provider: MarkerProvider): + '''Test check provider_check.''' marker_provider.mark(f"check:{marker_provider.label}") return True """ @@ -361,6 +371,7 @@ def provider_check(ctx, marker_provider: MarkerProvider): @loader def raw_provider_events(ctx, marker_provider: MarkerProvider): + '''Test loader raw_provider_events.''' marker_provider.mark(f"load:{marker_provider.label}") return [{"event_id": 1, "label": marker_provider.label}] """ @@ -370,6 +381,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): """ sources: - name: raw_provider_events + description: Test source raw_provider_events. managed: true write_strategy: table columns: @@ -441,6 +453,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): @task def failing_task(ctx, marker_provider: MarkerProvider): + '''Test task failing_task.''' marker_provider.mark("task") raise RuntimeError("intentional provider failure") """ @@ -463,6 +476,7 @@ def failing_task(ctx, marker_provider: MarkerProvider): @asset def failing_asset(ctx, marker_provider: MarkerProvider): + '''Test asset failing_asset.''' marker_provider.mark("asset") raise RuntimeError("intentional provider failure") """ @@ -485,6 +499,7 @@ def failing_asset(ctx, marker_provider: MarkerProvider): @check(depends_on=()) def failing_check(ctx, marker_provider: MarkerProvider): + '''Test check failing_check.''' marker_provider.mark("check") raise RuntimeError("intentional provider failure") """ @@ -507,6 +522,7 @@ def failing_check(ctx, marker_provider: MarkerProvider): @loader def raw_provider_events(ctx, marker_provider: MarkerProvider): + '''Test loader raw_provider_events.''' marker_provider.mark("load") raise RuntimeError("intentional provider failure") """ @@ -516,6 +532,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): """ sources: - name: raw_provider_events + description: Test source raw_provider_events. managed: true write_strategy: table columns: @@ -587,11 +604,12 @@ def test_given_provider_project_when_running_compile_or_plan_then_provider_setup @task def provider_task(ctx, marker_provider: MarkerProvider): + '''Test task provider_task.''' marker_provider.mark("task") """ ).strip() + "\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, ) @@ -648,6 +666,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ @task def alias_task(ctx, marker_provider: AliasMarkerProvider): + '''Test task alias_task.''' marker_provider.mark("task") """ ).strip() @@ -704,7 +723,7 @@ def materialize( """ ).strip() + "\n", - "models/orders.sql": "MODEL (materialized marker_mat);\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized marker_mat);\n\nSELECT 1 AS id\n", }, ) @@ -759,6 +778,7 @@ def test_given_concurrent_provider_backed_nodes_when_running_command_then_share_ class ConcurrentMarkerProvider(Provider): + '''Test provider ConcurrentMarkerProvider.''' marker_path: str = {str(marker_path)!r} @property @@ -786,12 +806,14 @@ def mark(self, label): @loader def raw_alpha(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_alpha.''' concurrent_marker_provider.mark("alpha") return [{"event_id": 1}] @loader def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_beta.''' concurrent_marker_provider.mark("beta") return [{"event_id": 2}] """ @@ -801,12 +823,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): """ sources: - name: raw_alpha + description: Test source raw_alpha. managed: true write_strategy: table columns: - name: event_id type: INTEGER - name: raw_beta + description: Test source raw_beta. managed: true write_strategy: table columns: @@ -816,10 +840,10 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): ).strip() + "\n", "models/fact_alpha.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' + 'MODEL (description "Test model fact_alpha.", materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' ), "models/fact_beta.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_beta")\n' + 'MODEL (description "Test model fact_beta.", materialized table);\n\nSELECT * FROM __source("raw_beta")\n' ), }, ) @@ -880,12 +904,14 @@ def test_given_python_hooks_with_provider_when_building_then_hooks_use_provider_ @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' ctx.providers.marker_provider.mark("pre_ctx") marker_provider.mark("pre_injected") @hook def mark_post(hook_context, marker_provider: MarkerProvider): + '''Test hook mark_post.''' hook_context.providers["marker_provider"].mark("post_ctx") marker_provider.mark("post_injected") """ @@ -893,7 +919,7 @@ def mark_post(hook_context, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")], post_hooks [python("mark_post")] @@ -948,18 +974,20 @@ def test_given_view_model_python_hooks_with_provider_when_building_then_hooks_us @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("view_pre") @hook def mark_post(ctx, marker_provider: MarkerProvider): + '''Test hook mark_post.''' marker_provider.mark("view_post") """ ).strip() + "\n", "models/orders_view.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders_view.", materialized view, pre_hooks [python("mark_pre")], post_hooks [python("mark_post")] @@ -992,7 +1020,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/incremental_orders.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model incremental_orders.", materialized incremental, incremental_strategy delete_insert, cursor order_id, @@ -1020,7 +1048,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/hourly_activity.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model hourly_activity.", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -1046,7 +1074,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): extra_repo_files={ "models/fact_orders.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model fact_orders.", materialized table); SELECT 1 AS order_id, TIMESTAMP '2026-01-01 00:00:00' AS ordered_at """ @@ -1067,7 +1095,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/customer_snapshot.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model customer_snapshot.", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -1096,7 +1124,7 @@ def mark_post(ctx, marker_provider: MarkerProvider): model_relative_path="models/custom_orders.sql", model_sql=dedent( """ - MODEL ( + MODEL (description "Test model custom_orders.", materialized copy_table, pre_hooks [python("mark_hook")], post_hooks [python("mark_hook")] @@ -1149,6 +1177,7 @@ def test_given_materialization_hooks_with_provider_when_building_then_hooks_use_ @hook def mark_hook(ctx, marker_provider: MarkerProvider): + '''Test hook mark_hook.''' marker_provider.mark(f"{ctx.model_name}:{ctx.phase}") """ ).strip() @@ -1203,13 +1232,14 @@ def test_given_untyped_python_hook_provider_parameter_when_building_then_provide @hook def mark_pre(ctx, marker_provider): + '''Test hook mark_pre.''' marker_provider.mark("untyped") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1263,6 +1293,7 @@ def test_given_python_hook_with_provider_when_hook_fails_then_provider_tears_dow @hook def failing_pre_hook(ctx, marker_provider: MarkerProvider): + '''Test hook failing_pre_hook.''' marker_provider.mark("pre") raise RuntimeError("intentional provider hook failure") """ @@ -1270,7 +1301,7 @@ def failing_pre_hook(ctx, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("failing_pre_hook")] ); @@ -1324,6 +1355,7 @@ def test_given_python_post_hook_with_provider_when_hook_fails_then_provider_tear @hook def failing_post_hook(ctx, marker_provider: MarkerProvider): + '''Test hook failing_post_hook.''' marker_provider.mark("post") raise RuntimeError("intentional provider post hook failure") """ @@ -1331,7 +1363,7 @@ def failing_post_hook(ctx, marker_provider: MarkerProvider): + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, post_hooks [python("failing_post_hook")] ); @@ -1395,13 +1427,14 @@ def test_given_alias_imported_provider_annotation_on_hook_when_building_then_cli @hook def mark_pre(ctx, marker_provider: AliasMarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1451,6 +1484,7 @@ def test_given_provider_annotation_mismatch_on_hook_when_building_then_cli_print class OtherProvider(Provider): + '''Test provider OtherProvider.''' pass """ ).strip() @@ -1463,13 +1497,14 @@ class OtherProvider(Provider): @hook def mark_pre(ctx, marker_provider: OtherProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre")] ); @@ -1524,13 +1559,14 @@ def test_given_python_hook_kwarg_matches_provider_when_building_then_cli_prints_ @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre", marker_provider: "literal")] ); @@ -1609,13 +1645,14 @@ def test_given_python_hook_kwarg_matches_context_when_building_then_cli_prints_c @hook def mark_pre(ctx, marker_provider: MarkerProvider): + '''Test hook mark_pre.''' marker_provider.mark("pre") """ ).strip() + "\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized table, pre_hooks [python("mark_pre", {test_case.context_parameter_name}: "literal")] ); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 78b1284e7a..7cc11f4498 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -34,7 +34,7 @@ def project_toml(*, target_settings: str = "") -> str: _PROJECT_TOML: str = project_toml() _SOURCES_YML: str = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -44,17 +44,17 @@ def project_toml(*, target_settings: str = "") -> str: " - name: order_date\n" ) STG_ORDERS: str = ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model.',\n materialized view,\n);\n\n" "SELECT\n order_id,\n customer_id,\n amount,\n order_date\n" 'FROM __source("raw_orders")\n' ) FACT_ORDERS: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n o.order_id,\n o.customer_id,\n o.amount,\n o.order_date\n" 'FROM __ref("stg_orders") AS o\n' ) CUSTOMER_TOTALS: str = ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model.',\n materialized table,\n);\n\n" "SELECT\n customer_id,\n SUM(amount) AS total_amount\n" 'FROM __ref("fact_orders")\nGROUP BY customer_id\n' ) @@ -81,20 +81,21 @@ def project_toml(*, target_settings: str = "") -> str: } UDF_FILE: tuple[str, str] = ( "functions/sql/udf__is_large_order.sql", - "FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", + "FUNCTION (description 'Test function udf__is_large_order.'," + "\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount >= 100\n", ) CENTS_MACRO: dict[str, str] = { "models/staging/_sqlbuild/_macros/cents.py": ( 'def to_cents(expression: str) -> str:\n return f"({expression}) * 100"\n' ), "models/staging/stg_order_cents.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_cents.',\n materialized view,\n);\n\n" 'SELECT\n order_id,\n @to_cents("amount") AS amount_cents\nFROM __ref("stg_orders")\n' ), } ORDER_REFUNDS: dict[str, str] = { "models/staging/stg_order_refunds.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_refunds.',\n materialized view,\n);\n\n" 'SELECT\n order_id,\n @to_cents("amount") AS refund_cents\nFROM __ref("stg_orders")\n' ), } @@ -115,26 +116,28 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: return { "models/marts/order_amounts.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model order_amounts.',\n materialized view,\n);\n\n" f'SELECT amount FROM __ref("{first}")\nUNION ALL\n' f'SELECT amount FROM __ref("{second}")\n' ), "models/marts/order_amount_reads.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' + 'MODEL (description "Test model order_amount_reads.",' + '\n materialized view,\n);\n\nSELECT amount FROM __ref("order_amounts")\n' ), } ORDER_EXPORT: dict[str, str] = { "models/marts/order_export.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' + 'MODEL (description "Test model order_export.",' + '\n materialized view,\n);\n\nSELECT *\nFROM __ref("fact_orders")\n' ), } CUSTOMER_DECLARATIONS: dict[str, str] = { "seeds/customers.csv": "customer_id,customer_name\n10,Ada\n11,Bo\n", "seeds/customers.yml": ( "seeds:\n" - " - name: customers\n" + " - name: customers\n description: Test seed customers.\n" " columns:\n" " - name: customer_id\n" " type: INTEGER\n" @@ -147,7 +150,7 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: ), "sources/customers.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" ' expression: "SELECT 10 AS customer_id UNION ALL SELECT 11"\n' " columns:\n" " - name: customer_id\n" @@ -157,7 +160,8 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: } ORDER_SHAPE: dict[str, str] = { "models/marts/_sqlbuild/_schemas/order_shape.sql": ( - "SCHEMA (\n name order_shape,\n columns (\n customer_id (audits [relationships " + "SCHEMA (description 'Test schema.'," + "\n name order_shape,\n columns (\n customer_id (audits [relationships " '(to __ref("stg_orders"), field customer_id)]),\n ),\n);\n' ), "models/marts/fact_orders.sql": FACT_ORDERS.replace( @@ -170,7 +174,7 @@ def order_amount_union(*, first: str, second: str) -> dict[str, str]: "def staged_orders() -> str:\n return '__ref(\"stg_orders\")'\n" ), "models/staging/stg_order_count.sql": ( - "MODEL (\n materialized view,\n);\n\n" + "MODEL (description 'Test model stg_order_count.',\n materialized view,\n);\n\n" "SELECT COUNT(*) AS order_count\nFROM @staged_orders()\n" ), } @@ -203,7 +207,8 @@ def order_history_files(*, materialized: str, udf: bool) -> dict[str, str]: return { "models/marts/order_history.sql": ( - f"MODEL (\n{_HISTORY_HEADERS[materialized]});\n\n" + "MODEL (description 'Test model order_history.'," + f"\n{_HISTORY_HEADERS[materialized]});\n\n" f"SELECT\n order_id,\n amount,\n order_date{_UDF_COLUMN[udf]}\n" 'FROM __ref("stg_orders")\n' ), @@ -278,7 +283,7 @@ def declare_missing_column_origin(*, project_dir: Path) -> None: contents: str = path.read_text(encoding="utf-8") _ = path.write_text( contents.replace( - "MODEL (\n", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n", 1 + "MODEL (", "MODEL (\n columns (revenue (migrate_from gross_amount)),\n ", 1 ).replace(" order_date\n", " order_date,\n amount AS revenue\n", 1), encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 2f84b7dea5..c08a3bdda2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -200,3 +200,4 @@ class RulesCacheEditCase: description: str edit: Callable[[Path], None] expected_exit_code: int = 1 + expected_diagnostics_fragment: str = "" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 831de8a32c..80fa105a91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -9,6 +9,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) @@ -21,7 +22,9 @@ def write_unevaluated_rules_project( adapter: str = "duckdb", selected_rules: tuple[str, ...] = ("SQBRSQL035",), resource_path: str = "models/staging/orders.sql", - resource_template: str = "MODEL ({options});\nSELECT {expression} AS order_id", + resource_template: str = ( + "MODEL (description 'Test model orders.', {options});\nSELECT {expression} AS order_id" + ), extra_files: tuple[tuple[str, str], ...] = (), ) -> None: (project_dir / "sqlbuild_project.toml").write_text( @@ -30,7 +33,9 @@ def write_unevaluated_rules_project( ) models: Path = project_dir / "models/staging" models.mkdir(parents=True) - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n" + ) relative_path: str contents: str for relative_path, contents in extra_files: @@ -239,13 +244,14 @@ def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, st "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" ), "models/staging/stg_orders.sql": ( - "MODEL (\n materialized view,\n contract enforced,\n columns (\n" + "MODEL (description 'Test model stg_orders.',\n materialized view,\n" + " contract enforced,\n columns (\n" " order_id (type INTEGER, nullable false, audits [not_null]),\n" " amount (type DOUBLE),\n ),\n);\n\n" "SELECT CAST(1 AS INTEGER) AS order_id, CAST(10.5 AS DOUBLE) AS amount\n" ), "models/marts/order_totals.sql": ( - "MODEL (\n columns (\n" + "MODEL (description 'Test model order_totals.',\n columns (\n" " order_id (type INTEGER, nullable false, audits [not_null]),\n" " net_amount (type DOUBLE),\n ),\n);\n\n" "SELECT\n o.order_id,\n" @@ -371,3 +377,20 @@ def rule_cache_counts(project_dir: Path) -> tuple[int, int]: result: subprocess.CompletedProcess[str] = run_compile_cli(project_dir) timings: dict[str, int] = json.loads(result.stdout)["compile_timings"] return timings["rule_cache_hits"], timings["rule_cache_misses"] + + +def move_order_totals_description_to_path_defaults(root: Path) -> None: + """Drop the mart model's header description and supply it from `[path_defaults]`.""" + + replace_project_text( + root, + "models/marts/order_totals.sql", + "MODEL (description 'Test model order_totals.',\n", + "MODEL (\n", + ) + config: Path = root / "sqlbuild_project.toml" + config.write_text( + config.read_text(encoding="utf-8") + + '\n[path_defaults.marts]\ndescription = "Order totals per order"\n', + encoding="utf-8", + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py index 96611b12f6..9a4faa1789 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_cache_invalidation.py @@ -12,6 +12,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RulesCacheEditCase from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( INCREMENTAL_RULES_PROJECT, + move_order_totals_description_to_path_defaults, rule_cache_counts, rules_compile_outcome, write_project_files, @@ -36,12 +37,35 @@ root, "models/staging/stg_orders.sql", " contract enforced,\n", "" ), ), + RulesCacheEditCase( + "model_description_edited", + lambda root: replace_project_text( + root, + "models/staging/stg_orders.sql", + "Test model stg_orders.", + "Staged order rows.", + ), + ), + RulesCacheEditCase( + "model_description_removed", + lambda root: replace_project_text( + root, + "models/marts/order_totals.sql", + "MODEL (description 'Test model order_totals.',\n", + "MODEL (\n", + ), + expected_diagnostics_fragment='"code": "P010"', + ), + RulesCacheEditCase( + "model_description_moved_to_path_defaults", + move_order_totals_description_to_path_defaults, + ), RulesCacheEditCase( "model_added", lambda root: write_project_file( root, "models/marts/order_counts.sql", - 'MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n', + 'MODEL (description "Test model order_counts.");\n\nSELECT order_id FROM __ref("stg_orders")\n', ), ), RulesCacheEditCase( @@ -131,6 +155,7 @@ def test_given_warm_rules_cache_when_one_input_changes_then_compile_equals_uncac rewarmed: tuple[int, str, str] = rules_compile_outcome(project_dir) assert edited[0] == test_case.expected_exit_code + assert test_case.expected_diagnostics_fragment in edited[1] assert edited == oracle assert rewarmed == oracle assert rule_cache_counts(project_dir)[1] == 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py index 2efe6d27b9..ede09a9dba 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -89,7 +89,7 @@ ) _KEYED_SOURCE_YML: str = ( "sources:\n" - " - name: landing_orders\n" + " - name: landing_orders\n description: Test source landing_orders.\n" " schema: landing\n" " columns:\n" " - name: order_id\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index 2d169f4577..41516c64ab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -25,7 +25,7 @@ UnevaluatedResourceCase( "model guard", "models/staging/orders.sql", - "MODEL ();\nSELECT {expression} AS order_id", + "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", 0, ), UnevaluatedResourceCase( @@ -34,12 +34,17 @@ "AUDIT ();\nSELECT {expression} AS order_id " 'FROM __ref("orders") JOIN __ref("customers") USING (order_id)', expected_evaluated_models=2, - extra_files=(("models/staging/customers.sql", "MODEL ();\nSELECT 1 AS order_id\n"),), + extra_files=( + ( + "models/staging/customers.sql", + "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", + ), + ), ), UnevaluatedResourceCase( "model the compiler accepts but the Rules parser rejects", "models/staging/orders.sql", - "MODEL ();\nWITH items AS (SELECT [1, 2] AS quantities)\n" + "MODEL (description 'Test model orders.');\nWITH items AS (SELECT [1, 2] AS quantities)\n" "SELECT 1 AS order_id, list_transform(i.quantities, lambda q: q + 1) AS shipped\n" "FROM items AS i", expected_evaluated_models=0, @@ -120,7 +125,8 @@ def test_given_prefix_not_like_when_running_rules_then_evaluates_the_complete_mo write_unevaluated_rules_project( project_dir=tmp_path, resource_path="models/staging/orders.sql", - resource_template="MODEL (database warehouse, schema analytics);\n" + test_case.sql, + resource_template="MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql, adapter="snowflake", ) for _ in range(2): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py index 6461358d51..7a7e88507a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py @@ -22,7 +22,7 @@ files={ "functions/sql/customer_orders.sql": dedent( """ - FUNCTION ( + FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (customer_id INTEGER, order_id INTEGER) ); @@ -33,7 +33,7 @@ + "\n", "models/order_summary.sql": dedent( """ - MODEL (materialized table); + MODEL (description "Test model order_summary.", materialized table); WITH customer_ids AS ( SELECT 7 AS customer_id @@ -58,7 +58,7 @@ description="generic audit argument may reference its local base CTE", files={ "models/orders.sql": ( - "MODEL (\n materialized table,\n audits [\n order_group_rate (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n audits [\n order_group_rate (\n" ' evaluation_sql "SELECT 1 AS is_ok FROM base",\n' " thresholds (error (below 0.5)),\n" " ),\n ],\n);\n\nSELECT 1 AS order_id\n" @@ -93,7 +93,7 @@ description="opaque audit SQL does not hide a later dead CTE", files={ "models/orders.sql": ( - "MODEL (\n materialized table,\n audits [\n order_group_rate (\n" + "MODEL (description 'Test model orders.',\n materialized table,\n audits [\n order_group_rate (\n" ' evaluation_sql "SELECT 1 AS is_ok FROM base",\n' " thresholds (error (below 0.5)),\n" " ),\n ],\n);\n\nSELECT 1 AS order_id\n" @@ -129,10 +129,10 @@ description="test harness output keeps its helper CTE reachable", files={ "models/raw_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 7 AS order_id, 25 AS amount\n" + "MODEL (description 'Test model raw_orders.', materialized table);\n\nSELECT 7 AS order_id, 25 AS amount\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT * FROM __ref("raw_orders")\n' ), "tests/unit/orders.sql": dedent( """ @@ -157,7 +157,7 @@ description="ordinary audit parameters do not hide a dead CTE", files={ "models/orders.sql": ( - "MODEL (materialized table, audits [order_rows]);\n\nSELECT 1 AS order_id\n" + "MODEL (description 'Test model orders.', materialized table, audits [order_rows]);\n\nSELECT 1 AS order_id\n" ), "audits/generic/order_rows.sql": dedent( """ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 2ff0dbebef..77f6d2d840 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -77,17 +77,18 @@ def build_scenario_project_files() -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT\n" " id AS order_id,\n" " amount\n" 'FROM __source("raw_orders")\n' ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" "SELECT\n" " SUM(amount) AS total_amount\n" 'FROM __ref("orders")\n' @@ -106,7 +107,7 @@ def build_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), "tests/scenarios/nested/orders_assert_pass.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario orders_assert_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -117,7 +118,7 @@ def build_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), "tests/scenarios/order_totals_fail.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario order_totals_fail.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -142,16 +143,19 @@ def build_partial_fixture_scenario_project_files() -> dict[str, str]: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" " - name: status\n type: VARCHAR\n" ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __source("raw_orders")\n' + ), "tests/scenarios/partial_orders.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario partial_orders.');\n\n" "WITH\n" "__source__raw_orders AS (SELECT 1 AS order_id),\n" "__expected__orders AS (\n" @@ -170,13 +174,14 @@ def build_scenario_python_hooks_project_files() -> dict[str, str]: "from sqlbuild.hooks import hook\n\n" "@hook\n" "def log_scenario_model(ctx):\n" - " ctx.execute_sql(\n" + " '''Test hook log_scenario_model.'''\n ctx.execute_sql(\n" ' f"CREATE TABLE {ctx.destination.schema}.scenario_hook_log AS "\n' " f\"SELECT '{ctx.model_name}' AS model_name, '{ctx.phase}' AS phase\"\n" " )\n" ) repo_files["models/orders.sql"] = ( - 'MODEL (materialized table, post_hooks [python("log_scenario_model")]);\n\n' + 'MODEL (description "Test model orders.", ' + 'materialized table, post_hooks [python("log_scenario_model")]);\n\n' "SELECT\n" " id AS order_id,\n" " amount\n" @@ -209,7 +214,8 @@ def build_real_warehouse_local_replay_project_files( return { "sqlbuild_project.toml": project_toml, "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: raw\n table: raw_events\n" ), "models/event_rollup.sql": model_sql, f"tests/scenarios/{scenario_name}.sql": scenario_sql, @@ -222,16 +228,17 @@ def build_real_warehouse_remote_scenario_project_files(*, project_toml: str) -> return { "sqlbuild_project.toml": project_toml, "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: raw\n table: raw_events\n" ), "models/stg_events.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_events.', materialized view);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' "WHERE amount_cents >= 1000\n" ), "models/event_rollup.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model event_rollup.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " SUM(amount_cents) AS large_amount_cents,\n" @@ -240,7 +247,7 @@ def build_real_warehouse_remote_scenario_project_files(*, project_toml: str) -> "GROUP BY customer_id\n" ), "tests/scenarios/remote_event_rollup.sql": ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario remote_event_rollup.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py index 7c5aa935ef..6d4f51e748 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py @@ -26,7 +26,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB integer division in fixture, expected and assertion SQL", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 21 // 2 AS amount -- integer division\n" @@ -46,7 +46,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB escape strings between relation markers in assertion SQL", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -66,7 +66,7 @@ ScenarioAuthoredCheckSqlE2ETestCase( description="DuckDB escape string before the first relation marker", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -110,7 +110,7 @@ def test_given_dialect_specific_scenario_sql_when_testing_then_runs_authored_sql ScenarioUnresolvableCheckSqlE2ETestCase( description="DuckDB backslash before a quote leaves the assertion string unclosed", scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -175,7 +175,7 @@ def test_given_project_dialect_scenario_sql_when_replaying_locally_then_resolves del repo_files["tests/scenarios/order_totals_fail.sql"] del repo_files["tests/scenarios/nested/orders_assert_pass.sql"] repo_files[f"tests/scenarios/{test_case.scenario_name}.sql"] = ( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario orders_assert_pass.');\n\n" "WITH\n" f"__source__raw_orders AS (\n {test_case.source_fixture_sql}\n),\n" f"__assert__no_named_totals AS (\n {test_case.assertion_sql}\n)\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index e1234511e1..64c8fdf399 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -59,7 +59,9 @@ def test_given_unclosed_scenario_comment_when_running_cli_then_error_names_scena tmp_path: Path, ) -> None: repo_files: dict[str, str] = build_scenario_project_files() - repo_files["tests/scenarios/order_totals_pass.sql"] = "SCENARIO ();\nWITH /* unfinished" + repo_files["tests/scenarios/order_totals_pass.sql"] = ( + "SCENARIO (description 'Test scenario order_totals_pass.');\nWITH /* unfinished" + ) project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="scenario_comment", repo_files=repo_files ) @@ -192,13 +194,13 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "models/local_model_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_model_error.', materialized table);\n\n" "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_model_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_model_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -228,17 +230,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/is_large_order.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -276,17 +278,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/bad_sql_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", + "FUNCTION (description 'Test function bad_sql_function.', arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", ), ( "models/local_sql_function_setup_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_setup_error.', materialized table);\n\n" 'SELECT id, __udf("bad_sql_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_setup_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_setup_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -331,17 +333,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/bad_transpile_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function bad_transpile_function.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_transpile_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_transpile_error.', materialized table);\n\n" 'SELECT id, __udf("bad_transpile_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_transpile_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_transpile_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -374,18 +376,18 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n additional_project_files=( ( "functions/sql/sql_runtime_error.sql", - "FUNCTION (arguments (amount INTEGER), returns INTEGER);\n\n" + "FUNCTION (description 'Test function sql_runtime_error.', arguments (amount INTEGER), returns INTEGER);\n\n" "CAST('bad' AS INTEGER)\n", ), ( "models/local_sql_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("sql_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -422,17 +424,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -477,17 +479,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' schema="analytics",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_error.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -538,17 +540,17 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> int:\n" - ' raise ValueError("python udf exploded")\n', + ' """Test function main."""\n raise ValueError("python udf exploded")\n', ), ( "models/local_python_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("python_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -900,13 +902,13 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "models/local_model_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_model_error.', materialized table);\n\n" "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_model_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_model_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -936,17 +938,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/is_large_order.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function is_large_order.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -984,17 +986,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/bad_sql_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", + "FUNCTION (description 'Test function bad_sql_function.', arguments (amount INTEGER), returns BOOLEAN);\n\nmissing_col > 9\n", ), ( "models/local_sql_function_setup_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_setup_error.', materialized table);\n\n" 'SELECT id, __udf("bad_sql_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_setup_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_setup_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1039,17 +1041,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/bad_transpile_function.sql", - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", + "FUNCTION (description 'Test function bad_transpile_function.', arguments (amount INTEGER), returns BOOLEAN);\n\namount > 9\n", ), ( "models/local_sql_function_transpile_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_transpile_error.', materialized table);\n\n" 'SELECT id, __udf("bad_transpile_function")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_transpile_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_transpile_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1093,18 +1095,18 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte additional_project_files=( ( "functions/sql/sql_runtime_error.sql", - "FUNCTION (arguments (amount INTEGER), returns INTEGER);\n\n" + "FUNCTION (description 'Test function sql_runtime_error.', arguments (amount INTEGER), returns INTEGER);\n\n" "CAST('bad' AS INTEGER)\n", ), ( "models/local_sql_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_sql_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("sql_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_sql_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_sql_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1141,17 +1143,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_pass.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_pass.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_pass.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_pass.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1196,17 +1198,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' schema="analytics",\n' ")\n" "def main(amount: int | None) -> bool:\n" - " return amount is not None and amount > 9\n", + " '''Test function main.'''\n return amount is not None and amount > 9\n", ), ( "models/local_python_function_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_error.', materialized table);\n\n" 'SELECT id, __udf("is_large_order_py")(amount) AS is_large_order\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" @@ -1257,17 +1259,17 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ' runtime_version="3.11",\n' ")\n" "def main(amount: int | None) -> int:\n" - ' raise ValueError("python udf exploded")\n', + ' """Test function main."""\n raise ValueError("python udf exploded")\n', ), ( "models/local_python_function_runtime_error.sql", - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model local_python_function_runtime_error.', materialized table);\n\n" 'SELECT id, __udf("python_runtime_error")(amount) AS bad_value\n' 'FROM __source("raw_orders")\n', ), ( "tests/scenarios/local_python_function_runtime_error.sql", - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario local_python_function_runtime_error.');\n\n" "WITH\n" "__source__raw_orders AS (\n" " SELECT 1 AS id, 10 AS amount\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index fbfed4847a..e2af4ae838 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -78,7 +78,7 @@ def write_scope_performance_project( def _write_scope_model(*, model_dir: Path, model_index: int, macro_name: str) -> None: (model_dir / f"model_{model_index:05d}.sql").write_text( - f'MODEL();\nSELECT @{macro_name}("id") AS id\n', encoding="utf-8" + f'MODEL(description "Test model.");\nSELECT @{macro_name}("id") AS id\n', encoding="utf-8" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py index fc2717e3dd..b61c1428bb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -24,7 +24,7 @@ def test_given_minimal_project_when_running_scope_aliases_then_outputs_are_offli 'name = "scope_e2e"\nadapter = "duckdb"\n', encoding="utf-8" ) (project_dir / "models" / "orders.sql").write_text( - "MODEL();\nSELECT 1 AS id\n", encoding="utf-8" + "MODEL(description 'Test model orders.');\nSELECT 1 AS id\n", encoding="utf-8" ) first: subprocess.CompletedProcess[str] = run_scope_alias( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py index 5e1b5ac518..fdfdc81dd2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py @@ -104,7 +104,7 @@ def test_given_empty_typed_seed_fields_when_loading_then_persists_nulls( ), "seeds/schema.yml": ( "seeds:\n" - " - name: nullable_mappings\n" + " - name: nullable_mappings\n description: Test seed nullable_mappings.\n" " columns:\n" " - name: mapping_id\n" " type: INTEGER\n" @@ -161,11 +161,11 @@ def test_given_standalone_seeds_when_writing_json_file_then_assets_are_complete_ ), "seeds/schema.yml": ( "seeds:\n" - " - name: first_seed\n" + " - name: first_seed\n description: Test seed first_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" - " - name: second_seed\n" + " - name: second_seed\n description: Test seed second_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" @@ -225,7 +225,7 @@ def test_given_failing_standalone_seed_when_writing_json_file_then_failure_is_ho ), "seeds/schema.yml": ( "seeds:\n" - " - name: invalid_seed\n" + " - name: invalid_seed\n description: Test seed invalid_seed.\n" " columns:\n" " - name: id\n" " type: INTEGER\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py index 21fbabdc1d..4faf5f6b84 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py @@ -252,7 +252,8 @@ def prepare_snowflake_diff_project(*, tmp_path: Path) -> tuple[Path, str, str]: staging_dir: Path = models_dir / "staging" staging_dir.mkdir(parents=True, exist_ok=True) (staging_dir / "stg_orders.sql").write_text( - "MODEL (materialized table, unique_key [order_id]);\n\n" + "MODEL (description 'Test model stg_orders.', " + "materialized table, unique_key [order_id]);\n\n" "SELECT * FROM (" "SELECT 1 AS order_id, 1 AS customer_id, 100 AS amount_cents UNION ALL " "SELECT 2 AS order_id, 2 AS customer_id, 200 AS amount_cents" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py index 1a22fe110e..1b47a05b6a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_dlt_snowflake.py @@ -65,11 +65,12 @@ def test_given_dlt_source_when_loading_to_snowflake_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 07c685357e..63f13f6223 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -181,14 +181,14 @@ def test_given_python_result_when_running_check_on_snowflake_then_persists_node_ "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -262,12 +262,12 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" f" schema: {schema_name}\n" " table: raw_orders\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' ), }, ) @@ -393,7 +393,7 @@ def test_given_source_deferral_env_when_building_on_snowflake_then_reads_prod_an "sqlbuild_local.toml": build_snowflake_local_config(schema_name=dev_schema_name), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -406,10 +406,10 @@ def test_given_source_deferral_env_when_building_on_snowflake_then_reads_prod_an "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -544,7 +544,7 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s SnowflakeScenarioLocalReplayE2ETestCase( description="captures snowflake fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " DATE_TRUNC('DAY', event_ts) AS event_day,\n" @@ -554,7 +554,7 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s "GROUP BY customer_id, DATE_TRUNC('DAY', event_ts)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, TO_TIMESTAMP_NTZ('2026-01-01 08:15:00') " @@ -585,12 +585,12 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s description="reports snowflake local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -613,14 +613,14 @@ def test_given_existing_snapshot_targets_when_building_on_snowflake_then_apply_s description="reports snowflake local DuckDB execution failures as X608", scenario_name="local_execution_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " __sqb_missing_local_function(amount_cents) AS amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -1177,7 +1177,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_snowflake_t repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: append\n" " cursor_column: load_seq\n" @@ -1191,7 +1191,7 @@ def test_given_loader_schema_evolution_project_when_loading_twice_on_snowflake_t "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader raw_events.'''\n if ctx.current_cursor_value is None:\n" " return [{'event_id': 1, 'load_seq': 1}]\n" " return [{'event_id': 2, 'load_seq': 2, 'note': 'late-note'}]\n" ), @@ -1256,7 +1256,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1271,10 +1271,10 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1329,7 +1329,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1338,7 +1338,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1364,7 +1364,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1375,7 +1375,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1396,7 +1396,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1407,7 +1407,7 @@ def test_given_chained_loader_project_when_loading_on_snowflake_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1432,7 +1432,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_snowflake_the repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py index d926ca1c2d..c500293d0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_dlt_sqlserver.py @@ -69,11 +69,12 @@ def test_given_dlt_source_when_loading_to_sqlserver_then_table_is_materialized_a f' credentials: "sqlite:///{source_db_path}"\n' " resources:\n" " - name: raw_orders\n" + " description: Test source raw_orders.\n" " table: orders\n" " write_disposition: replace\n" ), "models/order_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_totals.', materialized table);\n\n" 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index 054236995a..e89226e2d3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -118,11 +118,11 @@ def test_given_managed_source_when_cloning_sqlserver_then_source_precedes_depend "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_customers(ctx):\n" - " return []\n" + " '''Test loader raw_customers.'''\n return []\n" ), "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -132,7 +132,7 @@ def test_given_managed_source_when_cloning_sqlserver_then_source_precedes_depend " type: VARCHAR\n" ), "models/stg_customers.sql": ( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model stg_customers.', materialized view);\n\n" 'SELECT customer_id, first_name FROM __source("raw_customers")\n' ), }, @@ -294,7 +294,7 @@ def test_given_sqlserver_targets_when_running_diff_then_all_modes_execute_end_to f'schema = "{dev_schema}"\n' ), "models/fact_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model fact_orders.',\n" " materialized table,\n" " unique_key [order_id],\n" " cursor order_id,\n" @@ -436,14 +436,14 @@ def test_given_python_result_when_running_check_on_sqlserver_then_persists_node_ "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" + " '''Test task produce_result.'''\n return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" ), "python/checks/results.py": ( "from sqlbuild.checks import check\n" "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" - " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" + " '''Test check check_produce_result.'''\n return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" ), }, ) @@ -559,7 +559,7 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta SqlServerScenarioLocalReplayE2ETestCase( description="captures SQL Server fixtures and replays transpilable SQL locally", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT\n" " customer_id,\n" " CAST(CAST(event_ts AS DATE) AS DATETIME2) AS event_day,\n" @@ -570,7 +570,7 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta "GROUP BY customer_id, CAST(CAST(event_ts AS DATE) AS DATETIME2)\n" ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, CAST('2026-01-01 08:15:00' AS DATETIME2)" @@ -601,12 +601,12 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta description="reports SQL Server local transpilation failures as X607", scenario_name="local_transpile_error", model_sql=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT customer_id, amount_cents\n" 'FROM __source("raw_events")\n' ), scenario_sql=( - "SCENARIO ();\n\n" + "SCENARIO (description 'Test scenario.');\n\n" "WITH\n" "__source__raw_events AS (\n" " SELECT 10 AS customer_id, 1500 AS amount_cents\n" @@ -709,7 +709,7 @@ def test_given_source_loader_project_when_building_on_sqlserver_then_model_reads ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -722,10 +722,10 @@ def test_given_source_loader_project_when_building_on_sqlserver_then_model_reads "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -781,7 +781,7 @@ def test_given_source_deferral_env_when_building_on_sqlserver_then_reads_prod_an ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -794,10 +794,10 @@ def test_given_source_deferral_env_when_building_on_sqlserver_then_reads_prod_an "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " return [{'order_id': 7, 'status': 'loaded-dev'}]\n" + " '''Test loader raw_orders.'''\n return [{'order_id': 7, 'status': 'loaded-dev'}]\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), }, ) @@ -1031,7 +1031,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" @@ -1046,10 +1046,10 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}, {'event_id': 2}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}, {'event_id': 2}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id FROM {events.destination} ORDER BY event_id'\n" " )\n" @@ -1109,7 +1109,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " next_seq = 1\n" " else:\n" " next_seq = ctx.current_cursor_value + 1\n" @@ -1118,7 +1118,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1144,7 +1144,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ],\n" ")\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1155,7 +1155,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1176,7 +1176,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " {'name': 'load_seq', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " if ctx.current_cursor_value is None:\n" + " '''Test loader fetch_events.'''\n if ctx.current_cursor_value is None:\n" " return [\n" " {'event_id': 1, 'amount': 100, 'load_seq': 1},\n" " {'event_id': 2, 'amount': 200, 'load_seq': 1},\n" @@ -1187,7 +1187,7 @@ def test_given_chained_loader_project_when_loading_on_sqlserver_then_runs_loader " ]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " cursor = ctx.query(\n" " f'SELECT event_id, amount FROM {events.destination} '\n" " 'ORDER BY event_id, amount'\n" @@ -1213,7 +1213,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_sqlserver_the repo_files=build_schema_behavior_project_files( source_yaml=( "sources:\n" - " - name: raw_events\n" + " - name: raw_events\n description: Test source raw_events.\n" " managed: true\n" " write_strategy: table\n" " columns:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 777eb95cc7..e980efa666 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -14,7 +14,7 @@ def build_dynamic_pivot_test_project_files() -> dict[str, str]: 'contract = "enforced"\n' ), "models/stg_order_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_amounts.',\n" " database analytics,\n" " schema staging,\n" " columns (\n" @@ -30,7 +30,7 @@ def build_dynamic_pivot_test_project_files() -> dict[str, str]: "CAST('US' AS VARCHAR) AS country\n" ), "models/customer_category_amounts.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " database analytics,\n" " schema mart,\n" " columns (\n" @@ -86,7 +86,7 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s f"sql_analysis = {sql_analysis_value}\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "SELECT\n" " id,\n" " @mocked_amount() AS amount,\n" @@ -96,7 +96,7 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s 'FROM __source("raw")' ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "SELECT\n" " id,\n" " amount + 1 AS adjusted,\n" @@ -115,7 +115,8 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s "def real_status() -> str:\n" " return \"'active'\"\n" ), - "sources/raw.yml": "sources:\n - name: raw\n schema: main\n table: raw\n", + "sources/raw.yml": "sources:\n - name: raw\n description: Test source raw.\n" + " schema: main\n table: raw\n", "tests/unit/test_chain.sql": ( "TEST();\n\n" "WITH\n" @@ -146,10 +147,17 @@ def build_parameterized_test_project_files() -> dict[str, str]: "[connection]\n" 'database = "parameter_case_demo.duckdb"\n' ), - "models/orders.sql": 'MODEL ();\n\nSELECT status FROM __source("raw_orders")\n', - "models/customers.sql": ('MODEL ();\n\nSELECT status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT status FROM __source("raw_orders")\n' + ), + "models/customers.sql": ( + 'MODEL (description "Test model customers.");\n\n' + 'SELECT status FROM __source("raw_orders")\n' + ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( 'TEST (name "order_status", parameters (status string), cases (' @@ -181,14 +189,14 @@ def build_table_function_fixture_project_files() -> dict[str, str]: 'database = "table_function_fixture_demo.duckdb"\n' ), "functions/sql/customer_orders.sql": ( - "FUNCTION (\n" + "FUNCTION (description 'Test function customer_orders.',\n" " arguments (customer_id INTEGER),\n" " returns table (order_id INTEGER)\n" ");\n\n" "SELECT customer_id * 100 AS order_id\n" ), "models/orders.sql": ( - "MODEL (columns (order_id (type INTEGER)));\n\n" + "MODEL (description 'Test model orders.', columns (order_id (type INTEGER)));\n\n" 'SELECT order_id FROM __table_fn("customer_orders")(7)\n' ), "tests/unit/test_orders.sql": ( @@ -212,9 +220,16 @@ def build_mock_boundary_test_project_files() -> dict[str, str]: "[connection]\n" 'database = "mock_boundary_demo.duckdb"\n' ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", - "models/int_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n'), - "models/fact_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("int_orders")\n'), + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT 1 AS order_id\n", + "models/int_orders.sql": ( + 'MODEL (description "Test model int_orders.");\n\n' + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/fact_orders.sql": ( + 'MODEL (description "Test model fact_orders.");\n\n' + 'SELECT order_id FROM __ref("int_orders")\n' + ), "tests/unit/test_fact_orders.sql": ( "TEST();\n\n" "WITH\n" @@ -243,8 +258,9 @@ def build_unsatisfied_leaf_test_project_files() -> dict[str, str]: files: dict[str, str] = build_chain_test_project_files(sql_analysis_enabled=True) files["sources/raw.yml"] = ( "sources:\n" - " - name: raw\n schema: main\n table: raw\n" - " - name: unused\n schema: main\n table: unused\n" + " - name: raw\n description: Test source raw.\n schema: main\n table: raw\n" + " - name: unused\n description: Test source unused.\n" + " schema: main\n table: unused\n" ) files["tests/unit/test_chain.sql"] = ( "TEST();\n\n" @@ -270,11 +286,12 @@ def build_missing_mock_columns_project_files() -> dict[str, str]: 'database = "missing_fixture_columns.duckdb"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT customer_id, status FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT customer_id, status FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -301,10 +318,13 @@ def build_partial_source_fixture_project_files() -> dict[str, str]: "[connection]\n" 'database = "partial_source_fixture.duckdb"\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __source("raw_orders")\n' + ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -345,9 +365,13 @@ def build_partial_ref_fixture_project_files() -> dict[str, str]: 'database = "partial_ref_fixture.duckdb"\n' ), "models/stg_orders.sql": ( - "MODEL ();\n\nSELECT CAST(1 AS INTEGER) AS order_id, CAST(NULL AS VARCHAR) AS status\n" + "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(NULL AS VARCHAR) AS status\n" + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __ref("stg_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __ref("stg_orders")\n'), "tests/unit/test_orders.sql": ( "TEST();\n\n" "WITH\n" @@ -371,16 +395,20 @@ def build_open_schema_ref_fixture_project_files() -> dict[str, str]: 'database = "open_schema_ref_fixture.duckdb"\n' ), "models/stg_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_orders.',\n" " columns (\n" " order_id (type INTEGER),\n" " ),\n" ");\n\n" 'SELECT * FROM __source("raw_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id, status FROM __ref("stg_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id, status FROM __ref("stg_orders")\n' + ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( "TEST();\n\n" @@ -403,7 +431,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: 'database = "recursive_ref_fixture.duckdb"\n' ), "models/stg_order_links.sql": ( - "MODEL (\n" + "MODEL (description 'Test model stg_order_links.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -413,7 +441,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: 'SELECT order_id, parent_order_id FROM __source("raw_order_links")\n' ), "models/order_roots.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_roots.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -436,7 +464,7 @@ def build_recursive_ref_fixture_project_files() -> dict[str, str]: ), "sources/raw_order_links.yml": ( "sources:\n" - " - name: raw_order_links\n" + " - name: raw_order_links\n description: Test source raw_order_links.\n" " schema: main\n" " table: raw_order_links\n" " columns:\n" @@ -468,12 +496,13 @@ def build_partial_seed_fixture_project_files() -> dict[str, str]: 'database = "partial_seed_fixture.duckdb"\n' ), "models/countries.sql": ( - 'MODEL ();\n\nSELECT country_code, country_name FROM __seed("country_codes")\n' + 'MODEL (description "Test model countries.");\n\n' + 'SELECT country_code, country_name FROM __seed("country_codes")\n' ), "seeds/country_codes.csv": "country_code,country_name\nUS,United States\n", "seeds/schema.yml": ( "seeds:\n" - " - name: country_codes\n" + " - name: country_codes\n description: Test seed country_codes.\n" " columns:\n" " - name: country_code\n type: VARCHAR\n nullable: false\n" " - name: country_name\n type: VARCHAR\n nullable: true\n" @@ -494,7 +523,9 @@ def build_irrelevant_omitted_column_project_files() -> dict[str, str]: """Build a fixture omitting a known nullable column outside the compiled closure.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' + ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" "WITH\n" @@ -510,7 +541,7 @@ def build_cte_partial_source_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "WITH staged AS (\n" ' SELECT order_id, status FROM __source("raw_orders")\n' ")\n" @@ -524,7 +555,8 @@ def build_clause_partial_source_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - 'MODEL ();\n\nSELECT order_id FROM __source("raw_orders") WHERE status IS NULL\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT order_id FROM __source("raw_orders") WHERE status IS NULL\n' ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" @@ -543,7 +575,9 @@ def build_star_partial_fixture_project_files() -> dict[str, str]: files["sources/raw_orders.yml"] = files["sources/raw_orders.yml"].replace( " columns:\n", " contract: enforced\n columns:\n" ) - files["models/orders.sql"] = 'MODEL ();\n\nSELECT * FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __source("raw_orders")\n' + ) return files @@ -592,7 +626,7 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: 'database = "untyped_null_fixture.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (order_year (type BIGINT)),\n" ");\n\n" @@ -601,7 +635,7 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -625,7 +659,7 @@ def build_contract_empty_fixture_project_files() -> dict[str, str]: files: dict[str, str] = build_partial_source_fixture_project_files() files["models/orders.sql"] = ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" @@ -668,7 +702,9 @@ def build_open_expected_empty_fixture_project_files() -> dict[str, str]: """Build an empty expected fixture without an authoritative model shape.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT * FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __source("raw_orders")\n' + ) files["tests/unit/test_orders.sql"] = ( "TEST();\n\n" "WITH\n" @@ -691,16 +727,17 @@ def build_qualified_star_other_relation_project_files() -> dict[str, str]: "[connection]\n" 'database = "qualified_star_other_relation.duckdb"\n' ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS stg_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.');\n\n" + "SELECT 1 AS stg_id\n", "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "SELECT stg.*, raw.status\n" 'FROM __ref("stg_orders") stg\n' 'JOIN __source("raw_orders") raw ON TRUE\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -723,10 +760,13 @@ def build_mixed_case_partial_fixture_project_files() -> dict[str, str]: """Build a completed fixture with an as-written mixed-case unquoted alias.""" files: dict[str, str] = build_partial_source_fixture_project_files() - files["models/orders.sql"] = 'MODEL ();\n\nSELECT status, note FROM __source("raw_orders")\n' + files["models/orders.sql"] = ( + 'MODEL (description "Test model orders.");\n\n' + 'SELECT status, note FROM __source("raw_orders")\n' + ) files["sources/raw_orders.yml"] = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -753,7 +793,7 @@ def build_invalid_partial_fixture_project_files( files: dict[str, str] = build_partial_source_fixture_project_files() files["sources/raw_orders.yml"] = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " contract: enforced\n" @@ -788,12 +828,12 @@ def build_incompatible_fixture_type_project_files(*, adapter_name: str) -> dict[ 'schema = "main"\n' ), "models/orders.sql": ( - 'MODEL (database "fixture_db", schema "main");\n\n' + 'MODEL (description "Test model orders.", database "fixture_db", schema "main");\n\n' 'SELECT item_ids FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -821,12 +861,12 @@ def build_complex_values_fixture_project_files(*, adapter_name: str) -> dict[str 'schema = "main"\n' ), "models/orders.sql": ( - 'MODEL (database "fixture_db", schema "main");\n\n' + 'MODEL (description "Test model orders.", database "fixture_db", schema "main");\n\n' 'SELECT order_id, mapping_text FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -880,11 +920,12 @@ def build_union_distinct_expected_project_files() -> dict[str, str]: 'database = "union_distinct_expected.duckdb"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT DISTINCT order_id FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\n' + 'SELECT DISTINCT order_id FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -917,11 +958,12 @@ def build_transformed_collection_project_files() -> dict[str, str]: 'database = "transformed_collection.duckdb"\n' ), "models/order_statuses.sql": ( - 'MODEL ();\n\nSELECT list(status) AS statuses FROM __source("raw_orders")\n' + 'MODEL (description "Test model order_statuses.");\n\n' + 'SELECT list(status) AS statuses FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -948,7 +990,7 @@ def build_cte_derived_output_fixture_project_files() -> dict[str, str]: 'database = "cte_derived_output_fixture.duckdb"\n' ), "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" "WITH raw_orders AS (\n" ' SELECT * FROM __source("raw_orders")\n' "), derived AS (\n" @@ -963,7 +1005,8 @@ def build_cte_derived_output_fixture_project_files() -> dict[str, str]: "SELECT order_id, normalized_status, loaded_at FROM final\n" ), "sources/raw_orders.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/test_orders.sql": ( "TEST();\n\n" @@ -1013,10 +1056,11 @@ def build_assertion_test_project_files(*, failing: bool) -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT\n" " id AS order_id,\n" " amount\n" @@ -1050,7 +1094,8 @@ def build_macro_test_project_files() -> dict[str, str]: 'def normalize_status(value: str) -> str:\n return f"LOWER(TRIM({value}))"\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" + "MODEL (description 'Test model orders.', " + "materialized table);\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" ), "tests/unit/test_normalize_status.sql": ( 'TEST (mode macro, name "normalizes_status");\n\n' @@ -1081,25 +1126,26 @@ def build_diamond_chain_test_project_files(*, sql_analysis_enabled: bool) -> dic 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT id AS order_id, customer_id, amount FROM __source("raw_orders")\n' ), "models/large_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model large_orders.', materialized table);\n\n" 'SELECT order_id, customer_id, amount FROM __ref("stg_orders") WHERE amount >= 10\n' ), "models/small_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model small_orders.', materialized table);\n\n" "WITH picked AS (\n" ' SELECT order_id, customer_id, amount FROM __ref("stg_orders") WHERE amount < 10\n' ")\n" "SELECT * FROM picked -- small orders only\n" ), "models/customer_order_mix.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_order_mix.', materialized table);\n\n" "SELECT customer_id,\n" ' (SELECT COUNT(*) FROM __ref("large_orders") AS l\n' " WHERE l.customer_id = c.customer_id) AS large_count,\n" @@ -1150,17 +1196,20 @@ def build_deep_shared_missing_mock_project_files(*, layers: int) -> dict[str, st ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_customers\n schema: main\n table: raw_customers\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_customers\n description: Test source raw_customers.\n" + " schema: main\n table: raw_customers\n" ), "models/orders_00_left.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders_00_left.', materialized table);\n\n" "SELECT o.order_id, o.amount\n" 'FROM __source("raw_orders") AS o\n' 'JOIN __source("raw_customers") AS c ON c.customer_id = o.customer_id\n' ), "models/orders_00_right.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, amount FROM __ref("orders_00_left")\n' + 'MODEL (description "Test model orders_00_right.", ' + 'materialized table);\n\nSELECT order_id, amount FROM __ref("orders_00_left")\n' ), } for layer in range(1, layers + 1): @@ -1168,7 +1217,7 @@ def build_deep_shared_missing_mock_project_files(*, layers: int) -> dict[str, st previous_right: str = f"orders_{layer - 1:02d}_right" for side in ("left", "right"): files[f"models/orders_{layer:02d}_{side}.sql"] = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT a.order_id, a.amount + b.amount AS amount\n" f'FROM __ref("{previous_left}") AS a\n' f'JOIN __ref("{previous_right}") AS b ON a.order_id = b.order_id\n' @@ -1201,10 +1250,11 @@ def build_expected_column_subset_project_files( 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "SELECT id AS order_id, customer_id, status, amount * 2 AS amount\n" 'FROM __source("raw_orders")\n' ), @@ -1290,14 +1340,15 @@ def build_helper_scope_project_files() -> dict[str, str]: 'materialized = "table"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" 'SELECT id AS order_id, amount FROM __source("raw_orders")\n' ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" 'SELECT order_id, amount * 2 AS amount FROM __ref("stg_orders")\n' ), } @@ -1394,11 +1445,13 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_events\n schema: main\n table: raw_events\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" ), "models/daily_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model daily_orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor order_date,\n" @@ -1412,7 +1465,7 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()\n" ), "models/order_lookback.sql": ( - "MODEL (\n" + "MODEL (description 'Test model order_lookback.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor scan_start,\n" @@ -1428,7 +1481,7 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str "WHERE order_date >= CAST(__cursor_start() AS DATE) - INTERVAL 7 DAY\n" ), "models/event_totals.sql": ( - "MODEL (\n" + "MODEL (description 'Test model event_totals.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor event_id,\n" @@ -1441,10 +1494,11 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str " AND event_id - 1 < __cursor_end() + 1\n" ), "models/plain_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT amount FROM __source("raw_orders")\n' + 'MODEL (description "Test model plain_orders.", ' + 'materialized table);\n\nSELECT amount FROM __source("raw_orders")\n' ), "models/hourly_orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model hourly_orders.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" @@ -1488,7 +1542,8 @@ def build_cte_scope_project_files(*, queries: tuple[str, ...], expected: str) -> 'name = "cte_scope"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "tests/unit/chain.sql": ( 'TEST (name "order_chain");\nWITH ' @@ -1497,5 +1552,5 @@ def build_cte_scope_project_files(*, queries: tuple[str, ...], expected: str) -> ), } for index, query in enumerate(queries): - files[f"models/orders_{index}.sql"] = f"MODEL ();\n{query}" + files[f"models/orders_{index}.sql"] = f"MODEL (description 'Test model.');\n{query}" return files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py index 994276ecfa..875899670b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py @@ -15,9 +15,9 @@ 'name = "escape_strings"\nadapter = "duckdb"\n\n' '[connection]\ndatabase = "escape_strings.duckdb"\n' ), - "models/stg_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id\n", "models/order_names.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model order_names.', materialized table);\n\n" "SELECT E'O\\'Brien' AS customer_name, order_id\n" 'FROM __ref("stg_orders")\n' ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py index 32378e33a7..469d09acdd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py @@ -27,7 +27,7 @@ ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -35,7 +35,7 @@ " - name: status\n type: VARCHAR\n" ), "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (description 'Test model orders.');\n\n" 'SELECT order_id, @normalize_status("status") AS status FROM __source("raw_orders")\n' ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py index 6066ef65c6..d880521376 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py @@ -30,8 +30,8 @@ def test_given_combined_sql_over_256k_when_testing_on_duckdb_then_passes( 'name = "large_sql_test"\nadapter = "duckdb"\n\n' '[connection]\ndatabase = "orders.duckdb"\n' ), - "models/raw_orders.sql": "MODEL ();\nSELECT 'orders' AS details\n", - "models/orders.sql": 'MODEL ();\nSELECT details FROM __ref("raw_orders")\n', + "models/raw_orders.sql": "MODEL (description 'Test model raw_orders.');\nSELECT 'orders' AS details\n", + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT details FROM __ref("raw_orders")\n', "tests/unit/test_orders.sql": ( 'TEST (name "large_orders");\nWITH __ref__raw_orders AS (\n' f"SELECT '{payload}' AS details\n), __expected__orders AS (\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py index 7084916745..2d5d6e4dc9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_upstream_function_fallback.py @@ -39,18 +39,18 @@ def test_given_upstream_function_when_compiling_and_testing_then_mocked_chain_ex '[connection]\ndatabase = "orders.duckdb"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "functions/sql/identity_value.sql": ( - "FUNCTION (arguments (value INTEGER), returns INTEGER);\nvalue\n" + "FUNCTION (description 'Test function identity_value.', arguments (value INTEGER), returns INTEGER);\nvalue\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n" 'SELECT __udf("identity_value")(order_id) AS order_id ' 'FROM __source("raw_orders")\n' ), "models/orders.sql": ( - 'MODEL (materialized table);\nSELECT * FROM __ref("stg_orders")\n' + 'MODEL (description "Test model orders.", materialized table);\nSELECT * FROM __ref("stg_orders")\n' ), "tests/unit/orders_case.sql": ( 'TEST (name "orders_case");\nWITH\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py index 9481008692..7b39be0dc4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py @@ -112,7 +112,7 @@ def add_dbt_profile_downstream_model(*, sqlbuild_project_dir: Path) -> None: models_dir: Path = sqlbuild_project_dir / "models" models_dir.mkdir(exist_ok=True) (models_dir / "downstream_orders.sql").write_text( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model downstream_orders.', materialized table);\n\n" 'SELECT order_id FROM __dbt_ref("analytics", "dbt_orders")\n', encoding="utf-8", ) @@ -523,7 +523,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "sqlbuild_project.toml": project_toml, "models/current_customers.sql": build_current_customers_model_sql(plan="basic"), "models/current_customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region_id],\n" " snapshot_strategy timestamp,\n" @@ -541,7 +541,8 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st 'FROM __ref("current_customers")\n' ), "models/historical_customer_extracts.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model historical_customer_extracts.', " + "materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at, " "CAST('2026-01-02 00:00:00' AS TIMESTAMP) AS observed_at\n" @@ -553,7 +554,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "CAST('2026-01-04 00:00:00' AS TIMESTAMP) AS observed_at\n" ), "models/historical_customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -565,7 +566,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st 'FROM __ref("historical_customer_extracts")\n' ), "models/historical_membership_daily.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model historical_membership_daily.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS observed_at\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status, " @@ -578,7 +579,7 @@ def build_real_warehouse_snapshot_project_files(*, project_toml: str) -> dict[st "CAST('2026-01-03 00:00:00' AS TIMESTAMP) AS observed_at\n" ), "models/historical_membership_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_membership_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -600,7 +601,7 @@ def build_current_customers_model_sql(*, plan: str, updated_at: str = "2026-01-0 """Build the mutable current-state model for real-warehouse snapshot tests.""" return ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 10 AS region_id, " f"'{plan}' AS plan, CAST('{updated_at}' AS TIMESTAMP) AS updated_at\n" ) @@ -615,7 +616,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/current_check_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_check_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -628,7 +629,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/current_delete_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model current_delete_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -642,7 +643,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - changed=False ), "models/historical_timestamp_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_timestamp_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -656,7 +657,7 @@ def build_real_warehouse_existing_snapshot_project_files(*, project_toml: str) - ), "models/historical_check_daily.sql": build_historical_check_daily_model_sql(changed=False), "models/historical_check_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model historical_check_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -676,12 +677,12 @@ def build_current_check_customers_model_sql(*, changed: bool) -> str: return { False: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status\n" ), True: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'paused' AS status\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status\n" ), @@ -693,14 +694,14 @@ def build_current_delete_customers_model_sql(*, changed: bool) -> str: return { False: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at\n" "UNION ALL SELECT 2 AS customer_id, 'trial' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at\n" ), True: ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'pro' AS plan, " "CAST('2026-01-03 00:00:00' AS TIMESTAMP) AS updated_at\n" ), @@ -711,7 +712,7 @@ def build_historical_timestamp_extracts_model_sql(*, changed: bool) -> str: """Build mutable historical timestamp source model for live apply tests.""" sql: str = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'basic' AS plan, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at, " "CAST('2026-01-02 00:00:00' AS TIMESTAMP) AS observed_at\n" @@ -734,7 +735,7 @@ def build_historical_check_daily_model_sql(*, changed: bool) -> str: """Build mutable historical check source model for live apply tests.""" sql: str = ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "SELECT 1 AS customer_id, 'active' AS status, " "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS observed_at\n" "UNION ALL SELECT 2 AS customer_id, 'active' AS status, " diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index a7d343e087..f6eacfa60a 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -51,7 +51,7 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) (factories_dir / "generated_nodes.py").write_text( @@ -68,10 +68,12 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: "def generated_nodes():", " @task(depends_on=model('orders'), tags=['daily'], group='python')", " def prepare_orders(ctx):", + " '''Test task prepare_orders.'''", " return ctx.result(payload={'rows': 1})", "", " @loader(name='warehouse_export', depends_on=(prepare_orders,))", " def warehouse_export(ctx):", + " '''Test loader warehouse_export.'''", " return [{'order_id': 1}]", "", " @asset(", @@ -81,10 +83,12 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: " columns=[{'name': 'order_id', 'type': 'integer'}],", " )", " def orders_export(ctx):", + " '''Test asset orders_export.'''", " return ctx.result(materialized=True)", "", " @check(depends_on=orders_export, tags=['quality'], group='exports')", " def check_orders_export(ctx):", + " '''Test check check_orders_export.'''", " return True", "", " return [prepare_orders, warehouse_export, orders_export, check_orders_export]", diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index d6b60417fb..ee0fa7932e 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -103,7 +103,7 @@ def test_given_model_level_column_audits_when_loading_dagster_assets_then_checks 'database = "column_audit_checks.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " columns (\n" " order_id (),\n" " customer_id (),\n" @@ -543,14 +543,14 @@ def test_given_chained_source_loader_when_dagster_selects_source_then_reuses_int " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) setup_result: subprocess.CompletedProcess[str] = run_sqb( @@ -634,14 +634,14 @@ def test_given_chained_source_loader_when_dagster_selects_source_without_interme " {'name': 'event_id', 'type': 'INTEGER'},\n" "])\n" "def fetch_events(ctx):\n" - " return [{'event_id': 1}]\n\n" + " '''Test loader fetch_events.'''\n return [{'event_id': 1}]\n\n" "@loader(depends_on=[fetch_events])\n" "def raw_events(ctx):\n" - " events = ctx.loader(fetch_events)\n" + " '''Test loader raw_events.'''\n events = ctx.loader(fetch_events)\n" " ctx.execute_sql(f'CREATE OR REPLACE TABLE {ctx.destination} AS " "SELECT event_id FROM {events.destination}')\n" ), - "sources/raw.yml": "sources:\n - name: raw_events\n managed: true\n", + "sources/raw.yml": "sources:\n - name: raw_events\n description: Test source raw_events.\n managed: true\n", }, ) command_log_path: Path = tmp_path / "sqb_command_log.txt" diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py index 58c00bed17..cbb700083b 100644 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py +++ b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py @@ -55,7 +55,7 @@ def test_given_postgres_source_when_loading_with_ingestr_then_duckdb_target_has_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_pg_orders\n" + " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" " ingestr:\n" f' source_uri: "{postgres_uri(postgres_config)}"\n' f" source_table: public.{source_table}\n" @@ -113,7 +113,7 @@ def test_given_duckdb_source_when_loading_with_ingestr_to_duckdb_then_target_has ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" " ingestr:\n" f' source_uri: "duckdb:///{source_duckdb_path}"\n' " source_table: orders\n" @@ -175,7 +175,7 @@ def test_given_ingestr_duckdb_source_when_building_with_load_then_model_reads_lo ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -186,7 +186,7 @@ def test_given_ingestr_duckdb_source_when_building_with_load_then_model_reads_lo " source_table: orders\n" ), "models/stg_duckdb_orders.sql": """ -MODEL (materialized table); +MODEL (description "Test model stg_duckdb_orders.", materialized table); SELECT order_id, status FROM __source("raw_duckdb_orders") """.strip() @@ -250,7 +250,7 @@ def test_given_ingestr_source_when_building_with_load_then_model_reads_loaded_so ), "sources/raw.yml": ( "sources:\n" - " - name: raw_pg_orders\n" + " - name: raw_pg_orders\n description: Test source raw_pg_orders.\n" " columns:\n" " - name: order_id\n" " type: INTEGER\n" @@ -261,7 +261,7 @@ def test_given_ingestr_source_when_building_with_load_then_model_reads_loaded_so f" source_table: public.{source_table}\n" ), "models/stg_pg_orders.sql": """ -MODEL (materialized table); +MODEL (description "Test model stg_pg_orders.", materialized table); SELECT order_id, status FROM __source("raw_pg_orders") """.strip() @@ -327,7 +327,7 @@ def test_given_duckdb_source_when_loading_with_ingestr_then_postgres_target_has_ ), "sources/raw.yml": ( "sources:\n" - " - name: raw_duckdb_orders\n" + " - name: raw_duckdb_orders\n description: Test source raw_duckdb_orders.\n" f" schema: {target_schema}\n" " ingestr:\n" f' source_uri: "duckdb:///{source_duckdb_path}"\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 5abecd9b99..9287a1e455 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -357,13 +357,15 @@ class FormatScopeIntegrationTestCase: @dataclass(frozen=True) -class FormatWarningIntegrationTestCase: - """One non-failing format warning expectation.""" +class FormatDescriptionFaultIntegrationTestCase: + """A missing description faults format check and write mode alike.""" description: str + authored_sql: str expected_exit_code: int expected_code: str expected_severity: str + expected_formatted_sql: str @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index bb4537f6df..100649dbf9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -24,7 +24,7 @@ REPLAY_FULL: str = " replay_on_change full,\n" RELATION: str = f"main.{MODEL_NAME}" _INCREMENTAL_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy {strategy},\n" " unique_key order_id,\n" @@ -36,7 +36,7 @@ ");\n\n" ) _SNAPSHOT_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [order_id],\n" " snapshot_strategy timestamp,\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py index d047d909fb..42315ffa80 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_safety.py @@ -137,7 +137,7 @@ def test_given_unsafe_declared_rename_when_building_then_build_stops_before_chan ColumnMigrationCompileErrorTestCase( description="table materialization", model_sql=( - "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + "MODEL (description 'Test model.', materialized table, columns (revenue (migrate_from amount)));\n" 'SELECT amount AS revenue FROM __source("raw_orders")\n' ), expected_fragment="only valid for incremental and snapshot models", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 1c7ef28b7b..900532fb1c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -38,7 +38,8 @@ def write_compile_startup_project(project_dir: Path) -> None: models_dir: Path = project_dir / "models" models_dir.mkdir() (models_dir / "orders.sql").write_text( - "MODEL (materialized table);\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\nSELECT 1 AS order_id\n", + encoding="utf-8", ) @@ -110,7 +111,7 @@ def prepare_contract_project(tmp_path: Path, *, prod_connection_toml: str = "") models.mkdir() sources.mkdir() _ = (models / "orders.sql").write_text( - """MODEL ( + """MODEL (description "Test model orders.", materialized table -- keep model metadata columns ( @@ -240,10 +241,13 @@ def write_retention_policy_project(*, project_dir: Path, target_lines: tuple[str encoding="utf-8", ) _ = (project_dir / "models" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + encoding="utf-8", ) _ = (project_dir / "models" / "order_view.sql").write_text( - 'MODEL (materialized view);\n\nSELECT order_id FROM __ref("orders")\n', encoding="utf-8" + 'MODEL (description "Test model order_view.", ' + 'materialized view);\n\nSELECT order_id FROM __ref("orders")\n', + encoding="utf-8", ) @@ -292,10 +296,10 @@ def write_dropped_relation_project( ) _ = (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n", encoding="utf-8", @@ -305,7 +309,7 @@ def write_dropped_relation_project( ) _ = (project_dir / "seeds" / "lookups.yml").write_text( "seeds:\n" - " - name: order_statuses\n" + " - name: order_statuses\n description: Test seed order_statuses.\n" " columns:\n" " - name: status_code\n" " type: INTEGER\n" @@ -394,7 +398,7 @@ def dropped_relation_microbatch_sql(*, batch_concurrency: int) -> str: """Render an integer-cursor microbatch model over the raw order source.""" return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" @@ -419,7 +423,9 @@ def compile_duckdb_projection( (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text(f"MODEL ();\nSELECT {projection}") + (models / "orders.sql").write_text( + f"MODEL (description 'Test model orders.');\nSELECT {projection}" + ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) @@ -439,7 +445,7 @@ def write_expression_source_project( ) (project_dir / "sources" / "orders.yml").write_text( "sources:\n" - " - name: typed_orders\n" + " - name: typed_orders\n description: Test source typed_orders.\n" f' expression: "{source_expression}"\n' " columns:\n" " - name: order_id\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 72d48a5e89..b5aac79091 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -96,6 +96,7 @@ """ sources: - name: raw_orders + description: Test source raw_orders. schema: main table: raw_orders """ @@ -104,7 +105,7 @@ _FORCE_LINES: dict[bool, str] = {True: " migrate_force true,\n", False: ""} _DAILY_TOTALS_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_date,\n" @@ -116,7 +117,7 @@ ");\n\n" ) _SNAPSHOT_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [order_id],\n" " snapshot_strategy timestamp,\n" @@ -151,7 +152,7 @@ def incremental_orders_sql( migrate_from, f' migrate_from "{migrate_from}",\n' ) + _FORCE_LINES.get(migrate_force, "") return ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key order_id,\n" @@ -529,7 +530,7 @@ def enriched_view_sql(*, upstream: str, alias: str, materialized: str = "view") """Return a view or table that selects every order column through a table alias.""" return ( - f"MODEL (materialized {materialized});\n\n" + f"MODEL (description 'Test model.', materialized {materialized});\n\n" f"SELECT {alias}.order_id, {alias}.order_date, {alias}.amount_cents " f'FROM __ref("{upstream}") AS {alias}\n' ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py index f0eaac192f..14ac0ae2c9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_manual_migrations.py @@ -461,7 +461,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="force on a table", model_sql=( - 'MODEL (materialized table, migrate_from "stg_orders", migrate_force true);\n' + 'MODEL (description "Test model.", materialized table, migrate_from "stg_orders", migrate_force true);\n' "SELECT 1 AS id\n" ), expected_fragment="migrate_force is only valid for incremental and snapshot models", @@ -469,7 +469,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="force on a view", model_sql=( - 'MODEL (materialized view, migrate_from "stg_orders", migrate_force true);\n' + 'MODEL (description "Test model.", materialized view, migrate_from "stg_orders", migrate_force true);\n' "SELECT 1 AS id\n" ), expected_fragment="migrate_force is only valid for incremental and snapshot models", @@ -477,7 +477,7 @@ def test_given_replay_on_change_full_when_migrating_then_destination_is_not_repl MigrationCompileErrorTestCase( description="column alias on a table that is not migrating", model_sql=( - "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + "MODEL (description 'Test model.', materialized table, columns (revenue (migrate_from amount)));\n" "SELECT 1 AS revenue\n" ), expected_fragment="or for table and view models that declare migrate_from themselves", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 63c7799123..5b95c8f51f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -34,7 +34,7 @@ ).lstrip() _FACT_ORDER: tuple[str, ...] = ("required", "origin_archived", "view_created", "view_dropped") _TABLE_SQL: str = ( - "MODEL (materialized table{extra});\n" + "MODEL (description 'Test model.', materialized table{extra});\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -206,7 +206,8 @@ def disabled_table_sql(*, migrate_from: str) -> str: """Return a table model that declares its rename with old-name views turned off.""" return ( - f"MODEL (materialized table, migrate_from {migrate_from}, old_name_view false);\n" + "MODEL (description 'Test model.', " + f"materialized table, migrate_from {migrate_from}, old_name_view false);\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -225,7 +226,7 @@ def claiming_view_models() -> dict[str, str]: return { DESTINATION_MODEL: table_sql(migrate_from=ORIGIN_MODEL), ORIGIN_MODEL: ( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT order_id, amount_cents * 2 AS doubled FROM __source("raw_orders")\n' ), } @@ -236,7 +237,7 @@ def aliased_orders_sql(*, alias: str, migrate_from: str | None = None) -> str: migration: str = {None: ""}.get(migrate_from, f", migrate_from {migrate_from}") return ( - f"MODEL (materialized table, alias {alias}{migration});\n" + f"MODEL (description 'Test model.', materialized table, alias {alias}{migration});\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ) @@ -264,7 +265,7 @@ def aliased_table_sql(*, extra_column: str = "") -> str: """Return the renamed table model with ``amount`` renamed to ``revenue`` in its old view.""" return ( - f"MODEL (materialized table, migrate_from {ORIGIN_MODEL}, " + f"MODEL (description 'Test model.', materialized table, migrate_from {ORIGIN_MODEL}, " "columns (revenue (migrate_from amount_cents)));\n" f'SELECT order_id, amount_cents AS revenue{extra_column} FROM __source("raw_orders")\n' ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py index ffb7d41daf..902a1c9a70 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py @@ -14,7 +14,7 @@ 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "demo.duckdb"\n' ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (\n" @@ -25,7 +25,7 @@ "SELECT 1 AS order_id, 5 AS amount\n" ), "models/order_totals.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, amount * 2 AS doubled FROM __ref("orders")\n' + 'MODEL (description "Test model order_totals.", materialized table);\n\nSELECT order_id, amount * 2 AS doubled FROM __ref("orders")\n' ), "tests/unit/test_order_totals.sql": ( "TEST ();\nWITH\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py index 63a25020f4..66a6e8826c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py @@ -49,12 +49,12 @@ def test_given_typed_dependency_when_compiling_cte_then_preserves_binding_and_ou sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" ' expression: "(SELECT CAST(1 AS BIGINT) AS order_id)"\n' " columns:\n - name: order_id\n type: BIGINT\n" ) (models / "selected_orders.sql").write_text( - "MODEL (columns (result (type BOOLEAN)));\n" + "MODEL (description 'Test model selected_orders.', columns (result (type BOOLEAN)));\n" 'WITH selected AS (SELECT order_id FROM __source("orders"))\n' f"SELECT {test_case.projection} AS result FROM selected\n" ) @@ -83,7 +83,7 @@ def test_given_multiple_typed_model_headers_when_compiling_then_cli_preserves_he models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "staged_orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model staged_orders.',\n" " materialized table,\n" " columns (order_id (type INTEGER, nullable false)),\n" ' pre_hooks [inline_sql("SELECT 1")],\n' @@ -92,7 +92,7 @@ def test_given_multiple_typed_model_headers_when_compiling_then_cli_preserves_he encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized view,\n" " tags [core, 'daily orders'],\n" " columns (order_id (type DECIMAL(10,2))),\n" @@ -141,7 +141,7 @@ def test_given_model_macro_when_compiling_with_sql_rules_then_expands_once_and_p " return 'NULL'\n" ) (tmp_path / "models" / "orders" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id WHERE 1 = @missing_value()\n" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id WHERE 1 = @missing_value()\n" ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -173,7 +173,7 @@ def test_given_unicode_model_when_compiling_twice_then_cli_preserves_unchanged_a model_path: Path = tmp_path / "models" / "products.sql" model_path.parent.mkdir() model_path.write_text( - "MODEL (materialized table);\nSELECT 'café' AS product_name \n\n", + "MODEL (description 'Test model products.', materialized table);\nSELECT 'café' AS product_name \n\n", encoding="utf-8", ) @@ -235,7 +235,7 @@ def test_given_supported_snowflake_aggregation_when_compiling_then_project_succe model: Path = tmp_path / "models" / "order_summary.sql" model.parent.mkdir() model.write_text( - f"MODEL (database warehouse, schema analytics);\n\n{test_case.query_sql}", + f"MODEL (description 'Test model order_summary.', database warehouse, schema analytics);\n\n{test_case.query_sql}", encoding="utf-8", ) @@ -279,7 +279,7 @@ def test_given_nullable_output_when_compiling_contract_then_only_schema_nullabil model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" f" columns ({test_case.column_sql}),\n" ");\n\n" @@ -319,7 +319,7 @@ def test_given_relative_project_directory_when_compiling_then_private_macro_is_v 'def order_id() -> str:\n return "CAST(1 AS INTEGER)"\n', encoding="utf-8" ) (project_dir / "models" / "orders" / "orders.sql").write_text( - "MODEL (materialized table);\nSELECT @order_id() AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\nSELECT @order_id() AS order_id\n", encoding="utf-8", ) monkeypatch.chdir(tmp_path) @@ -350,17 +350,17 @@ def test_given_sql_test_chain_when_compiling_then_native_planner_writes_complete models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "stg_orders.sql").write_text( - 'MODEL (materialized table);\nSELECT order_id, amount FROM __source("raw_orders")\n', + 'MODEL (description "Test model stg_orders.", materialized table);\nSELECT order_id, amount FROM __source("raw_orders")\n', encoding="utf-8", ) (models_dir / "orders.sql").write_text( - 'MODEL (materialized table);\nSELECT order_id, amount FROM __ref("stg_orders")\n', + 'MODEL (description "Test model orders.", materialized table);\nSELECT order_id, amount FROM __ref("stg_orders")\n', encoding="utf-8", ) sources_dir: Path = tmp_path / "sources" sources_dir.mkdir() (sources_dir / "raw_orders.yml").write_text( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n", + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n", encoding="utf-8", ) tests_dir: Path = tmp_path / "tests" / "unit" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py index 5ce7a0bf1e..b2d4e72943 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py @@ -68,7 +68,7 @@ def test_given_derived_native_facts_when_compiling_then_contract_and_execution_a models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type BIGINT, nullable false)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (order_id (type BIGINT, nullable false)));\n" + test_case.query_sql ) @@ -115,9 +115,15 @@ def test_given_star_over_inferred_upstream_when_compiling_then_downstream_bindin (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') models: Path = tmp_path / "models" models.mkdir() - (models / "staged_orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id") - (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") - (models / "order_readers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + (models / "staged_orders.sql").write_text( + "MODEL (description 'Test model staged_orders.');\nSELECT 1 AS order_id" + ) + (models / "wide_orders.sql").write_text( + f"MODEL (description 'Test model wide_orders.');\n{test_case.star_model_sql}" + ) + (models / "order_readers.sql").write_text( + f"MODEL (description 'Test model order_readers.');\n{test_case.downstream_sql}" + ) for _ in ("cold", "warm"): exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -154,16 +160,18 @@ def test_given_star_over_derived_table_when_compiling_then_lineage_matches_proje seeds.mkdir() (seeds / "order_labels.csv").write_text("order_id,label\n1,first\n") (seeds / "order_labels.yml").write_text( - "seeds:\n - name: order_labels\n columns:\n" + "seeds:\n - name: order_labels\n description: Test seed order_labels.\n columns:\n" " - name: order_id\n type: INTEGER\n" " - name: label\n type: VARCHAR\n" ) models: Path = tmp_path / "models" models.mkdir() (models / "staged_orders.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount" + "MODEL (description 'Test model staged_orders.');\nSELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount" + ) + (models / "wide_orders.sql").write_text( + f"MODEL (description 'Test model wide_orders.');\n{test_case.star_model_sql}" ) - (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") for _ in ("cold", "warm"): exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -226,11 +234,13 @@ def test_given_snapshot_reader_when_compiling_then_validity_columns_are_known( models: Path = tmp_path / "models" models.mkdir() (models / "customer_snapshot.sql").write_text( - "MODEL (materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, " + "MODEL (description 'Test model customer_snapshot.', materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, " f"updated_at updated_at{test_case.snapshot_config});\n" "SELECT 1 AS customer_id, 'pro' AS plan, TIMESTAMP '2026-01-01' AS updated_at" ) - (models / "current_customers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + (models / "current_customers.sql").write_text( + f"MODEL (description 'Test model current_customers.');\n{test_case.downstream_sql}" + ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -334,14 +344,14 @@ def test_given_input_and_output_share_name_when_compiling_then_lineage_keeps_inp sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: INTEGER\n" " - name: amount\n type: DOUBLE\n" " - name: status\n type: VARCHAR\n" ) (models / "order_totals.sql").write_text( - "MODEL (contract enforced, columns (order_id (type INTEGER), " + "MODEL (description 'Test model order_totals.', contract enforced, columns (order_id (type INTEGER), " "status (type VARCHAR), total (type DOUBLE)));\n" f"{test_case.query_prefix}" "SELECT order_id, " @@ -438,7 +448,7 @@ def test_given_bound_wildcard_or_using_join_when_compiling_then_facts_match_exec sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: BIGINT\n nullable: false\n" " - name: category\n type: VARCHAR\n" @@ -447,7 +457,8 @@ def test_given_bound_wildcard_or_using_join_when_compiling_then_facts_match_exec models: Path = tmp_path / "models" models.mkdir() (models / "selected_orders.sql").write_text( - f"MODEL (contract enforced, columns ({test_case.output_contract}));\n" + test_case.query_sql + f"MODEL (description 'Test model selected_orders.', contract enforced, columns ({test_case.output_contract}));\n" + + test_case.query_sql ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) @@ -490,7 +501,7 @@ def test_given_object_wildcard_when_compiling_then_lineage_preserves_payload_inp sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( - "sources:\n - name: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n contract: enforced\n" " expression: orders_input\n columns:\n" " - name: order_id\n type: BIGINT\n" " - name: amount\n type: DOUBLE\n" @@ -498,7 +509,7 @@ def test_given_object_wildcard_when_compiling_then_lineage_preserves_payload_inp models: Path = tmp_path / "models" models.mkdir() (models / "selected_orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type BIGINT, nullable true)));\n" + "MODEL (description 'Test model selected_orders.', contract enforced, columns (order_id (type BIGINT, nullable true)));\n" + test_case.query_sql ) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index f3c0e4990d..6b230cdad2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -39,9 +39,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu config_path.write_text(config, encoding="utf-8") models: Path = tmp_path / "models" models.mkdir() - header: str = ( - "MODEL (materialized table, contract enforced, columns (order_id (type INTEGER)));\n" - ) + header: str = "MODEL (description 'Test model.', materialized table, contract enforced, columns (order_id (type INTEGER)));\n" for index in range(test_case.model_count): (models / f"orders_{index:03}.sql").write_text( header + "SELECT CAST(1 AS INTEGER) AS order_id\n", encoding="utf-8" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py index 5e8ef31528..59d14755df 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_contract_command.py @@ -167,7 +167,7 @@ def test_given_parameterized_physical_type_when_generating_then_writes_valid_mod database: Path = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table);\n" + "MODEL (description 'Test model orders.', materialized table);\n" "SELECT CAST(1 AS INTEGER) AS id, CAST(1 AS DECIMAL(10,2)) AS amount\n", encoding="utf-8", ) @@ -289,7 +289,7 @@ def test_given_conflicting_contract_when_overwriting_then_physical_shape_replace _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - """MODEL ( + """MODEL (description "Test model orders.", materialized table columns ( id (type BIGINT, description "identifier") @@ -603,7 +603,7 @@ def test_given_cli_vars_when_writing_then_validation_recompile_uses_same_values( _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table);\n" + "MODEL (description 'Test model orders.', materialized table);\n" "SELECT CAST(@@identifier AS INTEGER) AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) @@ -647,7 +647,7 @@ def test_given_unresolved_additive_type_conflict_when_writing_then_file_is_not_r _ = prepare_contract_project(tmp_path) model_path: Path = tmp_path / "models" / "orders.sql" _ = model_path.write_text( - "MODEL (materialized table, columns (id (type BIGINT), name (type VARCHAR)));\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (type BIGINT), name (type VARCHAR)));\n" "SELECT CAST(1 AS INTEGER) AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) @@ -698,7 +698,7 @@ def test_given_relative_project_directory_when_diffing_contract_then_private_mac model_path: Path = project_dir / "models" / "orders" / "orders.sql" model_path.parent.mkdir() _ = model_path.write_text( - "MODEL (materialized table);\nSELECT @order_id() AS id, CAST('a' AS VARCHAR) AS name\n", + "MODEL (description 'Test model orders.', materialized table);\nSELECT @order_id() AS id, CAST('a' AS VARCHAR) AS name\n", encoding="utf-8", ) macro_path: Path = model_path.parent / "_sqlbuild" / "_macros" / "order_id.py" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 62ce0ec451..13e71a9007 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -53,7 +53,7 @@ def test_given_project_when_running_command_then_discovery_skips_eager_column_lo project_dir=tmp_path, source_expression="(SELECT 1 AS order_id, 'open' AS status)", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py index 4502280b6d..236231f93d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py @@ -28,7 +28,7 @@ ) _INCREMENTAL_SQL: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " unique_key id,\n" @@ -39,7 +39,7 @@ ");\n\n" 'SELECT id, ordered_at FROM __source("raw_orders")\n' ) -_TABLE_SQL: str = 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' +_TABLE_SQL: str = 'MODEL (description "Test model.", materialized table);\n\nSELECT id FROM __source("raw_orders")\n' _MICROBATCH_CONCURRENCY_SETTINGS: str = ( "\n[settings]\nconcurrency = 3\nmicrobatch_concurrency = true\n" ) @@ -69,7 +69,7 @@ ), DroppedRelationRecoveryTestCase( description="view", - model_sql='MODEL (materialized view);\n\nSELECT id FROM __source("raw_orders")\n', + model_sql='MODEL (description "Test model.", materialized view);\n\nSELECT id FROM __source("raw_orders")\n', drop_sql="DROP VIEW main.orders", expected_action="create_view", expected_rows=((1,), (2,), (3,)), @@ -77,7 +77,7 @@ DroppedRelationRecoveryTestCase( description="snapshot", model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized snapshot,\n" " unique_key [id],\n" " snapshot_strategy timestamp,\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index 531b135075..10c641c8ad 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -16,11 +16,11 @@ DescriptionFormatIntegrationTestCase, DollarQuoteFormatIntegrationTestCase, FormatCompileIntegrationTestCase, + FormatDescriptionFaultIntegrationTestCase, FormatPathArgumentsIntegrationTestCase, FormatSafetyIntegrationTestCase, FormatScopeIntegrationTestCase, FormatterDeclineIntegrationTestCase, - FormatWarningIntegrationTestCase, FromValuesFormatIntegrationTestCase, LayoutOnlyFormatIntegrationTestCase, LeadingCteCommentFormatIntegrationTestCase, @@ -35,7 +35,9 @@ write_snowflake_format_test, ) -_UNFORMATTED_ORDERS_SQL: str = "MODEL (materialized table);\nselect 1 as order_id\n" +_UNFORMATTED_ORDERS_SQL: str = ( + 'MODEL (description "Test model.", materialized table);\nselect 1 as order_id\n' +) _LAYOUT_ONLY_BODY: str = 'SELECT ALL\n order_id id,\n amount,\nFROM __source("raw_orders") o;\n' @@ -271,6 +273,7 @@ def test_given_table_function_argument_when_formatting_then_intrinsic_restores_a function.parent.mkdir(parents=True) function.write_text( """FUNCTION ( + description "Test function expand_order.", arguments (order_id INTEGER), returns table (order_id INTEGER) ); @@ -811,7 +814,8 @@ def test_given_format_selectors_when_scoping_then_paths_and_default_exclusions_a models: Path = tmp_path / "models" models.mkdir() (models / "customers.sql").write_text( - "MODEL (materialized table, columns (customer_id (type INTEGER),));\n" + 'MODEL (description "Customers.", materialized table, ' + "columns (customer_id (type INTEGER),));\n" "SELECT 1 AS customer_id\n", encoding="utf-8", ) @@ -871,17 +875,19 @@ def test_given_format_selectors_when_scoping_then_paths_and_default_exclusions_a @pytest.mark.parametrize( "test_case", [ - FormatWarningIntegrationTestCase( - description="missing model description warns without failing format", - expected_exit_code=0, + FormatDescriptionFaultIntegrationTestCase( + description="missing model description fails check and write modes", + authored_sql="MODEL (materialized table);\nselect 1 as order_id\n", + expected_exit_code=1, expected_code="description-present", - expected_severity="warning", + expected_severity="fault", + expected_formatted_sql="MODEL (materialized table);\nSELECT 1 AS order_id\n", ) ], ids=lambda case: case.description, ) -def test_given_missing_description_when_formatting_then_warning_does_not_fail( - test_case: FormatWarningIntegrationTestCase, +def test_given_missing_description_when_formatting_then_check_and_write_fault( + test_case: FormatDescriptionFaultIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -890,26 +896,29 @@ def test_given_missing_description_when_formatting_then_warning_does_not_fail( ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL (materialized table);\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text(test_case.authored_sql, encoding="utf-8") - write_exit: int = main(["--project-dir", str(tmp_path), "format"]) - _ = capsys.readouterr() text_exit: int = main(["--project-dir", str(tmp_path), "format", "--check"]) text_output: str = capsys.readouterr().out json_exit: int = main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) payload: dict[str, object] = json.loads(capsys.readouterr().out) - assert write_exit == test_case.expected_exit_code assert text_exit == test_case.expected_exit_code - assert f"warning[{test_case.expected_code}]" in text_output + assert f"error[{test_case.expected_code}]" in text_output assert json_exit == test_case.expected_exit_code - assert payload["faults"] == 0 - assert payload["warnings"] == 1 + assert payload["faults"] == 1 + assert payload["warnings"] == 0 violations: object = payload["violations"] assert isinstance(violations, list) assert violations[0]["code"] == test_case.expected_code assert violations[0]["severity"] == test_case.expected_severity + write_exit: int = main(["--project-dir", str(tmp_path), "format"]) + write_output: str = capsys.readouterr().out + assert write_exit == test_case.expected_exit_code + assert f"error[{test_case.expected_code}]" in write_output + assert model.read_text(encoding="utf-8") == test_case.expected_formatted_sql + @pytest.mark.parametrize( "test_case", @@ -999,7 +1008,7 @@ def test_given_file_paths_when_formatting_then_only_named_files_change( LeadingCteCommentFormatIntegrationTestCase( description="comment block before a middle CTE stays above it and compiles", authored_sql=( - "MODEL (materialized view);\n\n" + 'MODEL (description "Test model.", materialized view);\n\n' 'WITH\nupstream AS (\n SELECT *\n FROM __ref("raw_orders")\n),\n\n' "-- Explains the next CTE: line one,\n-- line two.\n" "distinct_rows AS (\n SELECT order_id, customer_id\n FROM upstream\n" @@ -1022,7 +1031,7 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe models: Path = tmp_path / "models" models.mkdir() (models / "raw_orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id, 2 AS customer_id\n", + 'MODEL (description "Test model raw_orders.", materialized view);\n\nSELECT 1 AS order_id, 2 AS customer_id\n', encoding="utf-8", ) model_path: Path = models / "distinct_orders.sql" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 6554f61b17..4cc8af7813 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -83,6 +83,7 @@ @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 1, "status": "loaded"}] """ @@ -91,6 +92,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -105,10 +107,11 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 7, "status": "loaded"}] """, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), } @@ -225,6 +228,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. columns: - name: order_id type: INTEGER @@ -238,7 +242,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_UNSELECTED_MANAGED_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_MISSING_PROJECT_FILES, "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' ), } @@ -270,6 +274,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -290,10 +295,12 @@ def raw_orders(ctx): ], ) def fetch_orders(ctx): + '''Test loader fetch_orders.''' return [{"order_id": 7, "status": "intermediate"}] @loader(depends_on=[fetch_orders]) def raw_orders(ctx): + '''Test loader raw_orders.''' rows = ctx.query( f"SELECT order_id, status FROM {ctx.loader(fetch_orders).destination}" ).fetchall() @@ -319,6 +326,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: external_raw table: raw_orders managed: true @@ -362,6 +370,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table expression: | @@ -382,7 +391,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_SQL_FUNCTION_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_PROJECT_FILES, "functions/sql/order_statuses.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_statuses.", returns table ( order_id INTEGER, status VARCHAR @@ -422,6 +431,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table audits: [source_status] @@ -443,7 +453,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_BUILD_WITH_FUNCTION_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_PROJECT_FILES, "functions/sql/order_statuses.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_statuses.", returns table ( order_id INTEGER, status VARCHAR @@ -475,7 +485,7 @@ def raw_orders(ctx): _SOURCE_DEFERRAL_SCENARIO_PROJECT_FILES: dict[str, str] = { **_SOURCE_DEFERRAL_MISSING_PROJECT_FILES, "tests/scenarios/source_mock_pass.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario source_mock_pass."); WITH __source__raw_orders AS ( @@ -492,7 +502,7 @@ def raw_orders(ctx): _BUILD_RUN_AUTO_LOAD_SELECTION_PROJECT_FILES: dict[str, str] = { **_BUILD_RUN_AUTO_LOAD_PROJECT_FILES, "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id, status FROM __ref("stg_orders")' ), } @@ -503,6 +513,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' raise RuntimeError("loader exploded") """, } @@ -514,6 +525,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = "reload" if ctx.is_reload else "incremental" return [{"order_id": 7, "status": status}] """, @@ -524,6 +536,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -532,6 +545,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_customers + description: Test source raw_customers. managed: true write_strategy: table columns: @@ -546,10 +560,11 @@ def raw_orders(ctx): @loader def raw_customers(ctx): + '''Test loader raw_customers.''' return [{"customer_id": 10, "name": "Ada"}] """, "models/stg_customers.sql": ( - 'MODEL (materialized table);\n\nSELECT customer_id, name FROM __source("raw_customers")' + 'MODEL (description "Test model stg_customers.", materialized table);\n\nSELECT customer_id, name FROM __source("raw_customers")' ), } @@ -558,6 +573,7 @@ def raw_customers(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true columns: - name: order_id @@ -571,13 +587,14 @@ def raw_customers(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.execute_sql( "CREATE OR REPLACE TABLE raw_orders AS " "SELECT 7 AS order_id, 'self-managed' AS status" ) """, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT order_id, status FROM __source("raw_orders")' ), } @@ -591,6 +608,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -605,6 +623,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -641,6 +660,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -649,6 +669,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -658,6 +679,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -665,6 +687,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -699,6 +722,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -713,6 +737,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -1859,6 +1884,7 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1873,6 +1899,7 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -1909,6 +1936,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1917,6 +1945,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -1926,6 +1955,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -1933,6 +1963,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -1967,6 +1998,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -1981,6 +2013,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -2071,6 +2104,7 @@ def test_given_source_loader_when_running_load_then_writes_source_table( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2085,6 +2119,7 @@ def test_given_source_loader_when_running_load_then_writes_source_table( @loader def raw_orders(ctx): + '''Test loader raw_orders.''' ctx.log("loading raw orders") return [ {"order_id": 1, "status": "placed"}, @@ -2121,6 +2156,7 @@ def raw_orders(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2129,6 +2165,7 @@ def raw_orders(ctx): - name: status type: VARCHAR - name: raw_events + description: Test source raw_events. managed: true write_strategy: table """.strip() @@ -2138,6 +2175,7 @@ def raw_orders(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"order_id": 3, "status": "selected"}] """, "python/loaders/raw_events.py": """ @@ -2145,6 +2183,7 @@ def raw_orders(ctx): @loader def raw_events(ctx): + '''Test loader raw_events.''' return [{"event_id": 99}] """, }, @@ -2179,6 +2218,7 @@ def raw_events(ctx): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table columns: @@ -2193,6 +2233,7 @@ def raw_events(ctx): @loader def raw_orders(ctx): + '''Test loader raw_orders.''' status = ":".join([ str(ctx.target), str(ctx.vars["tier"]), @@ -2320,6 +2361,7 @@ def test_given_source_loader_when_running_pipeline_then_drops_stale_staging_firs "sources/raw.yml": """ sources: - name: raw_append_events + description: Test source raw_append_events. managed: true write_strategy: append cursor_column: event_id @@ -2335,6 +2377,7 @@ def test_given_source_loader_when_running_pipeline_then_drops_stale_staging_firs @loader def raw_append_events(ctx): + '''Test loader raw_append_events.''' if ctx.current_cursor_value is None: return [{"event_id": 1, "cursor_seen": "none"}] next_id = int(ctx.current_cursor_value) + 1 @@ -2351,6 +2394,7 @@ def raw_append_events(ctx): "sources/raw.yml": """ sources: - name: raw_merge_customers + description: Test source raw_merge_customers. managed: true write_strategy: merge unique_key: customer_id @@ -2369,6 +2413,7 @@ def raw_append_events(ctx): @loader def raw_merge_customers(ctx): + '''Test loader raw_merge_customers.''' if ctx.current_cursor_value is None: return [ {"customer_id": 1, "name": "old", "updated_at": 1}, @@ -2392,6 +2437,7 @@ def raw_merge_customers(ctx): "sources/raw.yml": """ sources: - name: raw_merge_composite + description: Test source raw_merge_composite. managed: true write_strategy: merge unique_key: [entity_id, source] @@ -2412,6 +2458,7 @@ def raw_merge_customers(ctx): @loader def raw_merge_composite(ctx): + '''Test loader raw_merge_composite.''' if ctx.current_cursor_value is None: return [{"entity_id": 1, "source": "api", "value": "old", "version": 1}] return [ @@ -2432,6 +2479,7 @@ def raw_merge_composite(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_events + description: Test source raw_delete_insert_events. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2447,6 +2495,7 @@ def raw_merge_composite(ctx): @loader def raw_delete_insert_events(ctx): + '''Test loader raw_delete_insert_events.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "old-outside-low"}, @@ -2477,6 +2526,7 @@ def raw_delete_insert_events(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_ts + description: Test source raw_delete_insert_ts. managed: true write_strategy: delete_insert cursor_column: event_at @@ -2494,6 +2544,7 @@ def raw_delete_insert_events(ctx): @loader def raw_delete_insert_ts(ctx): + '''Test loader raw_delete_insert_ts.''' if ctx.current_cursor_value is None: return [ {"event_at": datetime(2026, 1, 1, 0, 0, 0), "status": "outside-low"}, @@ -2525,6 +2576,7 @@ def raw_delete_insert_ts(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_empty + description: Test source raw_delete_insert_empty. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2540,6 +2592,7 @@ def raw_delete_insert_ts(ctx): @loader def raw_delete_insert_empty(ctx): + '''Test loader raw_delete_insert_empty.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "existing"}, @@ -2561,6 +2614,7 @@ def raw_delete_insert_empty(ctx): "sources/raw.yml": """ sources: - name: raw_no_cursor + description: Test source raw_no_cursor. managed: true write_strategy: append columns: @@ -2577,6 +2631,7 @@ def raw_delete_insert_empty(ctx): @loader def raw_no_cursor(ctx): + '''Test loader raw_no_cursor.''' global run_count run_count += 1 return [{"run_number": run_count, "cursor_seen": str(ctx.current_cursor_value)}] @@ -2592,6 +2647,7 @@ def raw_no_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_empty_cursor + description: Test source raw_empty_cursor. managed: true write_strategy: append cursor_column: event_id @@ -2607,6 +2663,7 @@ def raw_no_cursor(ctx): @loader def raw_empty_cursor(ctx): + '''Test loader raw_empty_cursor.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -2622,6 +2679,7 @@ def raw_empty_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_missing_cursor_target + description: Test source raw_missing_cursor_target. managed: true write_strategy: append cursor_column: event_id @@ -2637,6 +2695,7 @@ def raw_empty_cursor(ctx): @loader def raw_missing_cursor_target(ctx): + '''Test loader raw_missing_cursor_target.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -2653,6 +2712,7 @@ def raw_missing_cursor_target(ctx): "sources/raw.yml": """ sources: - name: raw_reload_context + description: Test source raw_reload_context. managed: true write_strategy: table columns: @@ -2665,6 +2725,7 @@ def raw_missing_cursor_target(ctx): @loader def raw_reload_context(ctx): + '''Test loader raw_reload_context.''' return [{"is_reload": ctx.is_reload}] """, } @@ -2674,6 +2735,7 @@ def raw_reload_context(ctx): "sources/raw.yml": """ sources: - name: raw_cursor_overrides + description: Test source raw_cursor_overrides. managed: true write_strategy: table columns: @@ -2692,6 +2754,7 @@ def raw_reload_context(ctx): @loader def raw_cursor_overrides(ctx): + '''Test loader raw_cursor_overrides.''' return [{ "start_ts": None if ctx.start_cursor_ts is None else ctx.start_cursor_ts.isoformat(), "end_ts": None if ctx.end_cursor_ts is None else ctx.end_cursor_ts.isoformat(), @@ -2712,6 +2775,7 @@ def raw_cursor_overrides(ctx): "sources/raw.yml": """ sources: - name: raw_append_events + description: Test source raw_append_events. managed: true write_strategy: append cursor_column: event_id @@ -2727,6 +2791,7 @@ def raw_cursor_overrides(ctx): @loader def raw_append_events(ctx): + '''Test loader raw_append_events.''' if ctx.current_cursor_value is None: return [{"event_id": 1, "cursor_seen": "none"}] next_id = int(ctx.current_cursor_value) + 1 @@ -2743,6 +2808,7 @@ def raw_append_events(ctx): "sources/raw.yml": """ sources: - name: raw_merge_customers + description: Test source raw_merge_customers. managed: true write_strategy: merge unique_key: customer_id @@ -2761,6 +2827,7 @@ def raw_append_events(ctx): @loader def raw_merge_customers(ctx): + '''Test loader raw_merge_customers.''' if ctx.current_cursor_value is None: return [ {"customer_id": 1, "name": "old", "updated_at": 1}, @@ -2784,6 +2851,7 @@ def raw_merge_customers(ctx): "sources/raw.yml": """ sources: - name: raw_merge_composite + description: Test source raw_merge_composite. managed: true write_strategy: merge unique_key: [entity_id, source] @@ -2804,6 +2872,7 @@ def raw_merge_customers(ctx): @loader def raw_merge_composite(ctx): + '''Test loader raw_merge_composite.''' if ctx.current_cursor_value is None: return [{"entity_id": 1, "source": "api", "value": "old", "version": 1}] return [ @@ -2824,6 +2893,7 @@ def raw_merge_composite(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_events + description: Test source raw_delete_insert_events. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2839,6 +2909,7 @@ def raw_merge_composite(ctx): @loader def raw_delete_insert_events(ctx): + '''Test loader raw_delete_insert_events.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "old-outside-low"}, @@ -2869,6 +2940,7 @@ def raw_delete_insert_events(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_ts + description: Test source raw_delete_insert_ts. managed: true write_strategy: delete_insert cursor_column: event_at @@ -2886,6 +2958,7 @@ def raw_delete_insert_events(ctx): @loader def raw_delete_insert_ts(ctx): + '''Test loader raw_delete_insert_ts.''' if ctx.current_cursor_value is None: return [ {"event_at": datetime(2026, 1, 1, 0, 0, 0), "status": "outside-low"}, @@ -2917,6 +2990,7 @@ def raw_delete_insert_ts(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_empty + description: Test source raw_delete_insert_empty. managed: true write_strategy: delete_insert cursor_column: event_id @@ -2932,6 +3006,7 @@ def raw_delete_insert_ts(ctx): @loader def raw_delete_insert_empty(ctx): + '''Test loader raw_delete_insert_empty.''' if ctx.current_cursor_value is None: return [ {"event_id": 1, "status": "existing"}, @@ -3220,6 +3295,7 @@ def delete_insert_cursor_spy( "sources/raw.yml": """ sources: - name: raw_no_cursor + description: Test source raw_no_cursor. managed: true write_strategy: append columns: @@ -3236,6 +3312,7 @@ def delete_insert_cursor_spy( @loader def raw_no_cursor(ctx): + '''Test loader raw_no_cursor.''' global run_count run_count += 1 return [{"run_number": run_count, "cursor_seen": str(ctx.current_cursor_value)}] @@ -3251,6 +3328,7 @@ def raw_no_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_empty_cursor + description: Test source raw_empty_cursor. managed: true write_strategy: append cursor_column: event_id @@ -3266,6 +3344,7 @@ def raw_no_cursor(ctx): @loader def raw_empty_cursor(ctx): + '''Test loader raw_empty_cursor.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -3281,6 +3360,7 @@ def raw_empty_cursor(ctx): "sources/raw.yml": """ sources: - name: raw_missing_cursor_target + description: Test source raw_missing_cursor_target. managed: true write_strategy: append cursor_column: event_id @@ -3296,6 +3376,7 @@ def raw_empty_cursor(ctx): @loader def raw_missing_cursor_target(ctx): + '''Test loader raw_missing_cursor_target.''' return [{"event_id": 1, "cursor_seen": str(ctx.current_cursor_value)}] """, }, @@ -3498,6 +3579,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context "sources/raw.yml": """ sources: - name: raw_a + description: Test source raw_a. managed: true write_strategy: table columns: @@ -3506,6 +3588,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context - name: connection_id type: BIGINT - name: raw_b + description: Test source raw_b. managed: true write_strategy: table columns: @@ -3514,6 +3597,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context - name: connection_id type: BIGINT - name: raw_c + description: Test source raw_c. managed: true write_strategy: table columns: @@ -3533,17 +3617,20 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context @loader def raw_a(ctx): + '''Test loader raw_a.''' barrier.wait(timeout=1) time.sleep(0.05) return [{"source_name": "raw_a", "connection_id": id(ctx.connection)}] @loader def raw_b(ctx): + '''Test loader raw_b.''' barrier.wait(timeout=1) return [{"source_name": "raw_b", "connection_id": id(ctx.connection)}] @loader def raw_c(ctx): + '''Test loader raw_c.''' return [{"source_name": "raw_c", "connection_id": id(ctx.connection)}] """, }, @@ -3617,6 +3704,7 @@ def test_given_multiple_source_loaders_when_running_pipeline_then_uses_concurren "sources/raw.yml": """ sources: - name: raw_join + description: Test source raw_join. managed: true write_strategy: table columns: @@ -3636,6 +3724,7 @@ def test_given_multiple_source_loaders_when_running_pipeline_then_uses_concurren @loader(write_strategy='table', columns=[{'name': 'id', 'type': 'INTEGER'}]) def fetch_a(ctx): + '''Test loader fetch_a.''' barrier.wait(timeout=1) time.sleep(0.05) with lock: @@ -3644,6 +3733,7 @@ def fetch_a(ctx): @loader(write_strategy='table', columns=[{'name': 'id', 'type': 'INTEGER'}]) def fetch_b(ctx): + '''Test loader fetch_b.''' barrier.wait(timeout=1) with lock: finished.add('b') @@ -3651,6 +3741,7 @@ def fetch_b(ctx): @loader(depends_on=[fetch_a, fetch_b]) def raw_join(ctx): + '''Test loader raw_join.''' return [{'upstream_count': len(finished)}] """, }, @@ -3738,6 +3829,7 @@ def test_given_loader_dag_when_running_pipeline_then_independent_branches_overla "sources/raw.yml": """ sources: - name: raw_inferred + description: Test source raw_inferred. managed: true write_strategy: table columns: @@ -3754,6 +3846,7 @@ def test_given_loader_dag_when_running_pipeline_then_independent_branches_overla @loader def raw_inferred(ctx): + '''Test loader raw_inferred.''' yield { "id": 1, "flag": True, @@ -3832,6 +3925,7 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri "sources/raw.yml": """ sources: - name: raw_multi_yield + description: Test source raw_multi_yield. managed: true write_strategy: table columns: @@ -3846,6 +3940,7 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri @loader def raw_multi_yield(ctx): + '''Test loader raw_multi_yield.''' yield {"id": 1, "status": "first"} yield {"id": 2, "status": "second"} """, @@ -3885,6 +3980,7 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ "sources/raw.yml": """ sources: - name: raw_batched_yield + description: Test source raw_batched_yield. managed: true write_strategy: table load_batch_size: 1 @@ -3900,6 +3996,7 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ @loader def raw_batched_yield(ctx): + '''Test loader raw_batched_yield.''' yield {"id": 1, "status": "first"} yield {"id": 2, "status": "second", "late_flag": True} yield {"id": 3, "status": "third", "late_flag": False} @@ -3979,6 +4076,7 @@ def test_given_generator_loader_uses_batch_size_when_running_pipeline_then_appen "sources/raw.yml": """ sources: - name: raw_missing_known + description: Test source raw_missing_known. managed: true write_strategy: table load_batch_size: 1 @@ -3994,6 +4092,7 @@ def test_given_generator_loader_uses_batch_size_when_running_pipeline_then_appen @loader def raw_missing_known(ctx): + '''Test loader raw_missing_known.''' yield {"id": 1, "status": "first"} yield {"id": 2} """, @@ -4011,6 +4110,7 @@ def raw_missing_known(ctx): "sources/raw.yml": """ sources: - name: raw_empty_generator + description: Test source raw_empty_generator. managed: true write_strategy: table load_batch_size: 1 @@ -4026,6 +4126,7 @@ def raw_missing_known(ctx): @loader def raw_empty_generator(ctx): + '''Test loader raw_empty_generator.''' if False: yield {"id": 1, "status": "unreachable"} """, @@ -4043,6 +4144,7 @@ def raw_empty_generator(ctx): "sources/raw.yml": """ sources: - name: raw_late_null + description: Test source raw_late_null. managed: true write_strategy: table load_batch_size: 1 @@ -4056,6 +4158,7 @@ def raw_empty_generator(ctx): @loader def raw_late_null(ctx): + '''Test loader raw_late_null.''' yield {"id": 1, "late_note": None} yield {"id": 2, "late_note": "filled"} """, @@ -4073,6 +4176,7 @@ def raw_late_null(ctx): "sources/raw.yml": """ sources: - name: raw_batch_size_two + description: Test source raw_batch_size_two. managed: true write_strategy: table load_batch_size: 2 @@ -4086,6 +4190,7 @@ def raw_late_null(ctx): @loader def raw_batch_size_two(ctx): + '''Test loader raw_batch_size_two.''' yield {"id": 1} yield {"id": 2} yield {"id": 3} @@ -4105,6 +4210,7 @@ def raw_batch_size_two(ctx): "sources/raw.yml": """ sources: - name: raw_default_batch + description: Test source raw_default_batch. managed: true write_strategy: table columns: @@ -4117,6 +4223,7 @@ def raw_batch_size_two(ctx): @loader def raw_default_batch(ctx): + '''Test loader raw_default_batch.''' yield {"id": 1} yield {"id": 2} yield {"id": 3} @@ -4194,6 +4301,7 @@ def test_given_batched_loader_variants_when_running_pipeline_then_writes_expecte "sources/raw.yml": """ sources: - name: raw_many_rows + description: Test source raw_many_rows. managed: true write_strategy: table columns: @@ -4206,6 +4314,7 @@ def test_given_batched_loader_variants_when_running_pipeline_then_writes_expecte @loader def raw_many_rows(ctx): + '''Test loader raw_many_rows.''' for value in range(1001): yield {"id": value} """, @@ -4255,6 +4364,7 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ "sources/raw.yml": """ sources: - name: raw_empty + description: Test source raw_empty. managed: true write_strategy: table columns: @@ -4269,6 +4379,7 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ @loader def raw_empty(ctx): + '''Test loader raw_empty.''' return [] """, }, @@ -4314,6 +4425,7 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec "sources/raw.yml": """ sources: - name: raw_self_managed + description: Test source raw_self_managed. managed: true columns: - name: id @@ -4325,6 +4437,7 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec @loader def raw_self_managed(ctx): + '''Test loader raw_self_managed.''' ctx.execute_sql("CREATE OR REPLACE TABLE raw_self_managed AS SELECT 1 AS id") """, }, @@ -4366,6 +4479,7 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta "sources/raw.yml": """ sources: - name: raw_conflict + description: Test source raw_conflict. managed: true write_strategy: table """.strip() @@ -4375,6 +4489,7 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta @loader def raw_conflict(ctx): + '''Test loader raw_conflict.''' return [ {"id": 1}, {"id": "two"}, @@ -4391,6 +4506,7 @@ def raw_conflict(ctx): "sources/raw.yml": """ sources: - name: raw_missing_strategy + description: Test source raw_missing_strategy. managed: true """.strip() + "\n", @@ -4399,6 +4515,7 @@ def raw_conflict(ctx): @loader def raw_missing_strategy(ctx): + '''Test loader raw_missing_strategy.''' return [{"id": 1}] """, }, @@ -4412,6 +4529,7 @@ def raw_missing_strategy(ctx): "sources/raw.yml": """ sources: - name: raw_unexpected_none + description: Test source raw_unexpected_none. managed: true write_strategy: table """.strip() @@ -4421,6 +4539,7 @@ def raw_missing_strategy(ctx): @loader def raw_unexpected_none(ctx): + '''Test loader raw_unexpected_none.''' return None """, }, @@ -4457,6 +4576,7 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle "sources/raw.yml": """ sources: - name: raw_batched_conflict + description: Test source raw_batched_conflict. managed: true write_strategy: table load_batch_size: 1 @@ -4467,6 +4587,7 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle @loader def raw_batched_conflict(ctx): + '''Test loader raw_batched_conflict.''' yield {"id": 1} yield {"id": "two"} """, @@ -4487,6 +4608,7 @@ def raw_batched_conflict(ctx): "sources/raw.yml": """ sources: - name: raw_batched_non_dict + description: Test source raw_batched_non_dict. managed: true write_strategy: table load_batch_size: 1 @@ -4497,6 +4619,7 @@ def raw_batched_conflict(ctx): @loader def raw_batched_non_dict(ctx): + '''Test loader raw_batched_non_dict.''' yield {"id": 1} yield ("id", 2) """, @@ -4512,6 +4635,7 @@ def raw_batched_non_dict(ctx): "sources/raw.yml": """ sources: - name: raw_append_failure + description: Test source raw_append_failure. managed: true write_strategy: append load_batch_size: 1 @@ -4525,6 +4649,7 @@ def raw_batched_non_dict(ctx): @loader def raw_append_failure(ctx): + '''Test loader raw_append_failure.''' yield {"id": 1} yield {"id": "two"} """, @@ -4545,6 +4670,7 @@ def raw_append_failure(ctx): "sources/raw.yml": """ sources: - name: raw_merge_failure + description: Test source raw_merge_failure. managed: true write_strategy: merge unique_key: id @@ -4561,6 +4687,7 @@ def raw_append_failure(ctx): @loader def raw_merge_failure(ctx): + '''Test loader raw_merge_failure.''' yield {"id": 1, "status": "new"} yield {"id": "two", "status": "bad"} """, @@ -4581,6 +4708,7 @@ def raw_merge_failure(ctx): "sources/raw.yml": """ sources: - name: raw_delete_insert_failure + description: Test source raw_delete_insert_failure. managed: true write_strategy: delete_insert cursor_column: id @@ -4597,6 +4725,7 @@ def raw_merge_failure(ctx): @loader def raw_delete_insert_failure(ctx): + '''Test loader raw_delete_insert_failure.''' yield {"id": 1, "status": "new"} yield {"id": "two", "status": "bad"} """, @@ -4671,6 +4800,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -4690,6 +4820,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4707,6 +4838,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -4726,9 +4858,11 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4773,6 +4907,7 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err "sources/raw.yml": """ sources: - name: raw_customers + description: Test source raw_customers. expression: SELECT 1 AS customer_id """.strip() + "\n", @@ -4794,6 +4929,7 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 33fb11fea3..80909ee47d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -117,14 +117,16 @@ def test_given_exact_internal_edge_exception_when_compiling_then_live_edge_passe models: Path = tmp_path / "models" models.mkdir() (models / "commerce__stg__orders.sql").write_text( - 'MODEL ();\nSELECT order_id FROM __ref("commerce__int_enriched__orders")\n', + 'MODEL (description "Test model commerce__stg__orders.");\nSELECT order_id FROM __ref("commerce__int_enriched__orders")\n', encoding="utf-8", ) (models / "commerce__int_enriched__orders.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model commerce__int_enriched__orders.');\nSELECT 1 AS order_id\n", + encoding="utf-8", ) (models / "commerce__mart__summary.sql").write_text( - "MODEL ();\nSELECT 1 AS order_count\n", encoding="utf-8" + "MODEL (description 'Test model commerce__mart__summary.');\nSELECT 1 AS order_count\n", + encoding="utf-8", ) live_exit: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) @@ -257,7 +259,7 @@ def test_given_constant_backed_decision_and_numeric_alias_when_compiling_then_ru model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - """MODEL ( + """MODEL (description "Test model orders.", constants (_large_batch 7), ); @@ -446,7 +448,7 @@ def test_given_enforced_contract_when_compiling_explicit_output_rule_then_enforc model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" f" columns ({test_case.columns_sql}),\n" ");\n\n" @@ -492,7 +494,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "stg_order_amounts.sql").write_text( - "MODEL (database analytics, schema analytics, columns (customer_id (type INTEGER), " + "MODEL (description 'Test model stg_order_amounts.', database analytics, schema analytics, columns (customer_id (type INTEGER), " "category (type VARCHAR), " 'amount (type "DECIMAL(12,2)")));\n' "SELECT CAST(1 AS INTEGER) AS customer_id, " @@ -502,7 +504,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ ) dynamic_model: Path = models_dir / "customer_category_amounts.sql" dynamic_model.write_text( - "MODEL (\n" + "MODEL (description 'Test model customer_category_amounts.',\n" " database analytics,\n" " schema analytics,\n" " columns (customer_id (type INTEGER)),\n" @@ -582,7 +584,7 @@ def test_given_contract_rules_when_compiling_then_contract_is_enforced_typed_and model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" f" columns ({test_case.columns_sql}),\n" ");\n\n" "SELECT CAST(1 AS INTEGER) AS order_id, CAST('ready' AS VARCHAR) AS status\n", @@ -1190,7 +1192,7 @@ def test_given_sql_function_argument_when_running_qualification_rule_then_argume function: Path = tmp_path / "functions" / "sql" / "orders_by_date.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function orders_by_date.", arguments (start_date DATE), returns table (order_id INTEGER) ); @@ -1236,7 +1238,7 @@ def test_given_table_function_argument_when_running_alias_rule_then_left_alias_i function: Path = tmp_path / "functions" / "sql" / "expand_order.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function expand_order.", arguments (order_id INTEGER), returns table (order_id INTEGER) ); @@ -1525,7 +1527,7 @@ def test_given_seed_and_function_references_when_running_source_rule_then_they_a function: Path = tmp_path / "functions" / "sql" / "normalize_quantity.sql" function.parent.mkdir(parents=True) function.write_text( - """FUNCTION ( + """FUNCTION (description "Test function normalize_quantity.", arguments (quantity INTEGER), returns INTEGER, ); @@ -2210,7 +2212,9 @@ def test_given_custom_rule_module_constant_edit_when_compiling_then_cached_resul ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_source: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule @@ -2259,8 +2263,12 @@ def test_given_focused_project_rule_result_when_compiling_full_project_then_subs ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") - (models / "customers.sql").write_text("MODEL ();\nSELECT 1 AS customer_id\n", encoding="utf-8") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + (models / "customers.sql").write_text( + "MODEL (description 'Test model customers.');\nSELECT 1 AS customer_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_file.write_text( @@ -2302,9 +2310,11 @@ def test_given_non_model_sql_violation_when_compiling_then_compile_is_authoritat ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) (tmp_path / "models" / "customers.sql").write_text( - "MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id\n", encoding="utf-8" ) audit: Path = tmp_path / "audits" / "singular" / "order_sample.sql" audit.parent.mkdir(parents=True) @@ -2339,9 +2349,12 @@ def test_given_exception_for_other_model_when_compiling_selection_then_exception ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) (models / "customers.sql").write_text( - "MODEL ();\nSELECT customer_id FROM customers LIMIT 1\n", encoding="utf-8" + "MODEL (description 'Test model customers.');\nSELECT customer_id FROM customers LIMIT 1\n", + encoding="utf-8", ) exit_code: int = main( @@ -2372,6 +2385,7 @@ def test_given_typed_unselected_model_when_planning_selection_then_rules_use_sel (sources / "customers.yml").write_text( """sources: - name: raw_customers + description: Test source raw_customers. schema: raw table: customers columns: @@ -2383,12 +2397,12 @@ def test_given_typed_unselected_model_when_planning_selection_then_rules_use_sel models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (contract enforced, columns (order_id (type INTEGER)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (order_id (type INTEGER)));\n" "SELECT CAST(1 AS INTEGER) AS order_id\n", encoding="utf-8", ) (models / "customers.sql").write_text( - """MODEL ( + """MODEL (description "Test model customers.", contract enforced, columns (customer_id (type VARCHAR)), ); @@ -2485,7 +2499,10 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai ) model: Path = project_dir / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT order_id FROM orders LIMIT 1\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT order_id FROM orders LIMIT 1\n", + encoding="utf-8", + ) monkeypatch.chdir(tmp_path) first_exit: int = main(["--project-dir", "project", "compile", "--json"]) @@ -2515,7 +2532,7 @@ def test_given_unused_inline_suppression_when_compiling_then_stale_directive_is_ model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL ();\n-- sqb: ignore SQBRSQL004 because this fixture checks stale directives\n" + "MODEL (description 'Test model orders.');\n-- sqb: ignore SQBRSQL004 because this fixture checks stale directives\n" "SELECT 1 AS order_id\n", encoding="utf-8", ) @@ -2544,7 +2561,9 @@ def test_given_unselected_custom_rule_options_when_compiling_then_configuration_ ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() - model.write_text("MODEL ();\nSELECT 1 AS order_id\n", encoding="utf-8") + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) rule_file: Path = tmp_path / "rules" / "architecture.py" rule_file.parent.mkdir() rule_file.write_text( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py index 0344d39ee5..4d42869376 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py @@ -35,7 +35,9 @@ def test_given_sql_and_custom_exceptions_when_compiling_cold_and_warm_then_both_ ) models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id LIMIT 1\n") + (models / "orders.sql").write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id LIMIT 1\n" + ) rules: Path = tmp_path / "rules" rules.mkdir() (rules / "architecture.py").write_text( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py b/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py index 52690a0815..c3da610ca9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_source_rebinding.py @@ -22,7 +22,7 @@ description="names already known at compile time add no rebinding", source_expression="(SELECT 1 AS order_id, 'open' AS status)", model_sql=( - "MODEL (materialized table);\n" + "MODEL (description 'Test model.', materialized table);\n" 'SELECT o.order_id, o.status FROM __source("typed_orders") AS o\n' ), expected_exit_code=0, @@ -32,7 +32,7 @@ description="names hidden from compile rebind and reject a missing column", source_expression="(SELECT * FROM main.raw_orders)", model_sql=( - 'MODEL (materialized table);\nSELECT o.missing FROM __source("typed_orders") AS o\n' + 'MODEL (description "Test model.", materialized table);\nSELECT o.missing FROM __source("typed_orders") AS o\n' ), expected_exit_code=1, expected_rebinding=True, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py b/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py index c9a273d56a..ed904eb422 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_target_retention_views.py @@ -23,13 +23,13 @@ [ TargetRetentionViewsTestCase( description="view skips inherited target retention", - view_header="MODEL (materialized view);", + view_header="MODEL (description 'Test model.', materialized view);", expected_exit_code=0, expected_fragment="Project compiled", ), TargetRetentionViewsTestCase( description="explicit view retention is still rejected", - view_header="MODEL (materialized view, time_travel_retention 7d);", + view_header="MODEL (description 'Test model.', materialized view, time_travel_retention 7d);", expected_exit_code=1, expected_fragment="managed time_travel_retention is not valid for views", ), @@ -44,7 +44,8 @@ def test_given_target_retention_default_when_compiling_views_then_only_explicit_ (tmp_path / "models").mkdir() (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") (tmp_path / "models" / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + encoding="utf-8", ) (tmp_path / "models" / "order_view.sql").write_text( f'{test_case.view_header}\n\nSELECT order_id FROM __ref("orders")\n', encoding="utf-8" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 77dce05d9b..b5dd9c4a9a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -61,7 +61,7 @@ class SemanticCompileCase: upstream: str downstream: str expected_code: str | None - header: str = "MODEL (materialized view);\n" + header: str = "MODEL (description 'Test model.', materialized view);\n" expected_diagnostic_count: int = 1 expected_column: int | None = None diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index debd19f9d3..906227cb77 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -156,9 +156,12 @@ def compile_reshaped_star_model( models: Path = project_dir / "models" models.mkdir() for name, sql in RESHAPED_STAR_UPSTREAM_MODELS.items(): - (models / f"{name}.sql").write_text(f"MODEL (materialized table);\n{sql}", encoding="utf-8") + (models / f"{name}.sql").write_text( + f"MODEL (description 'Test model.', materialized table);\n{sql}", encoding="utf-8" + ) (models / "reshaped_orders.sql").write_text( - f"MODEL (materialized table);\n{query_sql}", encoding="utf-8" + f"MODEL (description 'Test model reshaped_orders.', materialized table);\n{query_sql}", + encoding="utf-8", ) project: CompiledProject = compile_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), @@ -176,7 +179,9 @@ def lineage_source_pairs(column: CompiledLineageColumnFact) -> frozenset[tuple[s ) -_SHARED_BINDING_UPSTREAM_SQL: str = "MODEL (materialized view); SELECT 1 AS id, 2.5 AS amount" +_SHARED_BINDING_UPSTREAM_SQL: str = ( + "MODEL (description 'Test model.', materialized view); SELECT 1 AS id, 2.5 AS amount" +) def write_shared_binding_project(*, project_dir: Path, test_case: SharedBindingQueryCase) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py index e671bc3cd8..0a3f70c70f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py @@ -25,7 +25,7 @@ "sqlbuild_project.toml": ( 'name = "gate_demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n' ), - "models/valid_codes.sql": "MODEL (materialized table);\nSELECT 'A' AS code\n", + "models/valid_codes.sql": "MODEL (description 'Test model valid_codes.', materialized table);\nSELECT 'A' AS code\n", "audits/singular/reconcile.sql": ( 'AUDIT ();\nSELECT o.* FROM __ref("orders") o JOIN __ref("valid_codes") v USING (code)\n' ), @@ -37,13 +37,13 @@ [ SingularAuditAttachmentIntegrationTestCase( description="unrelated models without a gate edge run the singular audit at the end", - orders_header="MODEL (materialized table);", + orders_header="MODEL (description 'Test model.', materialized table);", generic_audit_files={}, expected_attachment=("end", None), ), SingularAuditAttachmentIntegrationTestCase( description="an attached audit's gate edge does not move the singular audit", - orders_header="MODEL (materialized table, audits [order_check]);", + orders_header="MODEL (description 'Test model.', materialized table, audits [order_check]);", generic_audit_files=_ORDER_CHECK, expected_attachment=("end", None), ), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py index 15d13ced77..103fd56042 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py @@ -46,7 +46,7 @@ def order_quality(): ] """, "models/orders.sql": """ -MODEL (audit_factories [order_quality]); +MODEL (description "Test model orders.", audit_factories [order_quality]); SELECT 10 AS amount """, }, @@ -57,7 +57,7 @@ def order_quality(): tmp_path, { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [ expression_is_true (name "positive_amount", expression "amount > 0", severity error), expression_is_true (name "bounded_amount", expression "amount < 1000", severity warn), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py index ce365a20bb..c4ff500ce8 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py @@ -28,12 +28,16 @@ def test_given_inferred_producer_when_binding_consumers_then_avoids_deferred_req ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS quantity\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS id, 2 AS quantity\n" ) (tmp_path / "models").mkdir() producer: Path = tmp_path / "models/orders.sql" - producer.write_text('MODEL (materialized view); SELECT id FROM __source("raw_orders")') - (tmp_path / "models/report.sql").write_text("MODEL (materialized view); " + test_case.sql) + producer.write_text( + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' + ) + (tmp_path / "models/report.sql").write_text( + "MODEL (description 'Test model report.', materialized view); " + test_case.sql + ) original: Callable[..., Any] = project.get_schema_validations requests: list[Any] = [] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py index 592026618f..3305168812 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py @@ -33,18 +33,20 @@ def test_given_completed_shape_when_an_unrelated_model_changes_then_cached_colum ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n table: orders\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n - name: quantity\n type: INTEGER\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n - name: quantity\n type: INTEGER\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/order_inputs.sql").write_text( - 'MODEL (materialized view); SELECT id, quantity FROM __source("raw_orders")' + 'MODEL (description "Test model order_inputs.", materialized view); SELECT id, quantity FROM __source("raw_orders")' ) (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view);\n" + "MODEL (description 'Test model orders.', materialized view);\n" + test_case.sql.replace('__source("raw_orders")', '__ref("order_inputs")') ) edited: Path = tmp_path / "models/customers.sql" - edited.write_text("MODEL (materialized view); SELECT 1 AS id") + edited.write_text( + "MODEL (description 'Test model customers.', materialized view); SELECT 1 AS id" + ) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] assert main(args) == 0 cold: dict[str, Any] = json.loads(capsys.readouterr().out) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index d88e58c44b..0719d710ad 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -24,19 +24,19 @@ '[targets.dev]\nschema = "dev_schema"\n' ), "models/staging/stg_orders.sql": ( - "MODEL (materialized table\ntags [staging]);\n\nSELECT 1 AS order_id" + "MODEL (description 'Test model stg_orders.', materialized table\ntags [staging]);\n\nSELECT 1 AS order_id" ), "models/staging/stg_customers.sql": ( - "MODEL (materialized table\ntags [staging]);\n\nSELECT 1 AS customer_id" + "MODEL (description 'Test model stg_customers.', materialized table\ntags [staging]);\n\nSELECT 1 AS customer_id" ), "models/intermediate/int_orders.sql": ( - 'MODEL (materialized table\ntags [core]);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model int_orders.", materialized table\ntags [core]);\n\nSELECT order_id FROM __ref("stg_orders")' ), "models/marts/fact_orders.sql": ( - 'MODEL (materialized table\ntags [core, mart]);\n\nSELECT order_id FROM __ref("int_orders")' + 'MODEL (description "Test model fact_orders.", materialized table\ntags [core, mart]);\n\nSELECT order_id FROM __ref("int_orders")' ), "models/marts/dim_customers.sql": ( - 'MODEL (materialized table\ntags [mart]);\n\nSELECT customer_id FROM __ref("stg_customers")' + 'MODEL (description "Test model dim_customers.", materialized table\ntags [mart]);\n\nSELECT customer_id FROM __ref("stg_customers")' ), } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index f72f6980c9..306d2278f3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -31,12 +31,12 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' ) (tmp_path / "sources").mkdir() source: Path = tmp_path / "sources/orders.yml" source.write_text( - f"sources:\n - name: raw_orders\n expression: {test_case.expression}\n - name: raw_customers\n expression: {test_case.expression}\n" + f"sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.expression}\n - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.expression}\n" ) original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch calls: list[object] = [] @@ -81,13 +81,13 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized view); SELECT id FROM __source("raw_orders")' ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( "sources:\n" - f" - name: raw_orders\n expression: {test_case.orders_expression}\n" - f" - name: raw_customers\n expression: {test_case.customers_expression}\n" + f" - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.orders_expression}\n" + f" - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.customers_expression}\n" ) original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch batches: list[tuple[str, ...]] = [] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 1d74739134..bf14bcf478 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -64,7 +64,7 @@ def test_given_active_adapter_when_macro_renders_constant_then_uses_adapter_sql( "def region_array(ctx) -> str:\n return ctx.render_constant('regions')\n" ), "models/summary.sql": ( - "MODEL (materialized view, database warehouse, schema analytics);\n\n" + "MODEL (description 'Test model summary.', materialized view, database warehouse, schema analytics);\n\n" "SELECT @region_array() AS regions" ), }, @@ -328,7 +328,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return str(ctx.constants['minimum_quantity'])\n" ), "models/summary.sql": ( - "MODEL (materialized view, audits [minimum_quantity]);\n\n" + "MODEL (description 'Test model summary.', materialized view, audits [minimum_quantity]);\n\n" "SELECT @minimum_quantity() AS minimum_quantity" ), "tests/unit/summary.sql": ( @@ -367,7 +367,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return str(ctx.constants['minimum_quantity'])\n" ), "models/products/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @minimum_quantity() AS minimum_quantity" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @minimum_quantity() AS minimum_quantity" ), }, expected_error_fragment="Constant 'minimum_quantity'.*is inaccessible", @@ -424,7 +424,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " return ctx.render_constant('maximum_quantity')\n" ), "models/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @missing_quantity() AS maximum_quantity" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @missing_quantity() AS maximum_quantity" ), }, expected_error_fragment=( @@ -445,7 +445,7 @@ def test_given_macro_context_declarations_when_compiling_resources_then_all_expa " )\n" ), "models/summary.sql": ( - "MODEL (materialized view);\n\nSELECT @missing_status() AS status" + "MODEL (description 'Test model summary.', materialized view);\n\nSELECT @missing_status() AS status" ), }, expected_error_fragment=( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index eb77e2825d..45ab004033 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -44,7 +44,7 @@ "sqlbuild_project.toml": _PROJECT_TOML, "models/_macros/counting.py": _COUNTING_MACRO_MODULE, "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS order_id, @order_tag() AS tag" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id, @order_tag() AS tag" ), }, expected_macro_import_count=1, @@ -91,7 +91,7 @@ def test_given_side_effect_macro_when_compiling_manifest_then_imports_macros_onc "def _helper() -> str:\n return shared()\n\n" "def order_column() -> str:\n return _helper()\n" ), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT @order_column()", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT @order_column()", }, macro_name="order_column", expected_dependencies=("shared",), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 2f6bdd4759..f7dee2a733 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -53,7 +53,7 @@ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT '12' AS amount\n" " columns:\n" " - name: amount\n" @@ -68,6 +68,7 @@ ), "models/orders.sql": ( "MODEL (\n" + ' description "Test model orders.",\n' " contract enforced,\n" " model_schema order_amount,\n" ");\n\n" @@ -153,7 +154,9 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i description="single table model with no schema defaults to adapter schema", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id" + ), }, expected_models={ "orders": ExpectedModelEntry( @@ -172,7 +175,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/active_orders.sql": ( - "MODEL (materialized view);\n\nSELECT order_id FROM orders WHERE status = 'active'" + "MODEL (description 'Test model active_orders.', materialized view);\n\nSELECT order_id FROM orders WHERE status = 'active'" ), }, expected_models={ @@ -192,7 +195,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": ( - "MODEL (\n materialized table\n schema analytics\n" + "MODEL (description 'Test model orders.',\n materialized table\n schema analytics\n" " database warehouse\n);\n\n" "SELECT 1 AS order_id" ), @@ -213,9 +216,11 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i description="two models with ref dependency resolves ref to qualified name", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/stg_orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/stg_orders.sql": ( + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id" + ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, expected_models={ @@ -242,10 +247,10 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _FILE_PROJECT_TOML, "sources/raw.yml": ( - "sources:\n - name: raw_payments\n schema: main\n table: payments\n" + "sources:\n - name: raw_payments\n description: Test source raw_payments.\n schema: main\n table: payments\n" ), "models/stg_payments.sql": ( - 'MODEL (materialized table);\n\nSELECT payment_id FROM __source("raw_payments")' + 'MODEL (description "Test model stg_payments.", materialized table);\n\nSELECT payment_id FROM __source("raw_payments")' ), }, expected_models={ @@ -265,10 +270,10 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/staging/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS order_id" + "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id" ), "models/marts/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, expected_models={ @@ -329,7 +334,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i " )\n" ), "models/policy_summary.sql": ( - "MODEL (materialized view);\n\nSELECT @policy_columns()" + "MODEL (description 'Test model policy_summary.', materialized view);\n\nSELECT @policy_columns()" ), }, expected_models={ @@ -423,12 +428,12 @@ def test_given_project_files_when_running_compile_pipeline_then_produces_valid_o description="run selector resolves Python task without SQL-only planning", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " return ctx.result(payload={'ok': True})\n" + " '''Test task prepare_orders.'''\n return ctx.result(payload={'ok': True})\n" ), }, expected_models={}, @@ -465,7 +470,7 @@ def test_given_python_run_selector_when_running_compile_pipeline_then_tracks_pyt description="reports compile progress from compile pipeline", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, expected_progress_prefixes=("Compiling project...", "Compiled project."), ) @@ -498,7 +503,7 @@ def test_given_progress_callback_when_running_compile_pipeline_then_reports_comp description="reports compile progress from project graph build", project_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, expected_progress_prefixes=("Compiling project...", "Compiled project."), ) @@ -543,7 +548,7 @@ def test_given_progress_callback_when_building_project_graph_then_reports_compil 'database = "TEST_DB"\n' 'schema = "TEST_SCHEMA"\n' ), - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, expected_error_fragment="snowflake execution requires explicit target database", ) @@ -585,7 +590,7 @@ def test_given_snowflake_local_override_without_target_namespace_when_compiling_ 'database = "${ENV:TEST_DB}"\n' 'schema = "${ENV:TEST_SCHEMA}"\n' ), - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", }, expected_database="LOCAL_DB", expected_schema="LOCAL_SCHEMA", @@ -633,9 +638,11 @@ def test_given_snowflake_local_target_namespace_when_compiling_then_targets_reso "[targets.prod]\n" 'schema = "prod_schema"\n' ), - "models/stg_orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS order_id"), + "models/stg_orders.sql": ( + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS order_id" + ), "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' + 'MODEL (description "Test model fact_orders.", materialized table);\n\nSELECT order_id FROM __ref("stg_orders")' ), }, defer_to="prod", @@ -696,15 +703,15 @@ def test_given_project_with_defer_to_when_compiling_then_resolves_refs_to_deferr "sql_analysis = true\n" ), "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, amount FROM __source("raw")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT id, amount FROM __source("raw")' ), "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model fact_orders.', materialized table);\n\n" "WITH local_helper AS (SELECT 1 AS one) " 'SELECT id, amount + one AS adjusted FROM __ref("stg_orders") ' "CROSS JOIN local_helper" ), - "sources/raw.yml": "sources:\n - name: raw\n schema: main\n table: raw\n", + "sources/raw.yml": "sources:\n - name: raw\n description: Test source raw.\n schema: main\n table: raw\n", "tests/unit/test_chain.sql": ( "TEST();\n\n" "WITH\n" @@ -796,6 +803,7 @@ def test_given_append_cursor_model_when_compiling_then_sql_uses_expected_lower_b db_path: Path = tmp_path / "append_cursor.duckdb" model_header: str = ( "MODEL (\n" + ' description "Test model orders.",\n' " materialized incremental,\n" " incremental_strategy append,\n" " cursor ordered_at,\n" @@ -812,7 +820,7 @@ def test_given_append_cursor_model_when_compiling_then_sql_uses_expected_lower_b f'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "{db_path}"\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n schema: main\n table: raw_orders\n" ), "models/orders.sql": model_header, }, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index 387bb23a98..f9d5697d63 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -44,7 +44,7 @@ def test_given_attached_measurement_audit_when_compiling_then_full_contract_reac 'database = ":memory:"\n' ), "models/orders.sql": """ - MODEL ( + MODEL (description "Test model orders.", materialized table, audits [ valid_order_rate ( @@ -138,7 +138,7 @@ def test_given_generic_measurement_definition_with_policy_when_compiling_then_de tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (audits [rate (thresholds (warn (below 100)))]); SELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.', audits [rate (thresholds (warn (below 100)))]); SELECT 1", "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate, thresholds (warn (below 100))); MEASURE (SELECT 100 AS rate FROM @relation); @@ -169,7 +169,7 @@ def test_given_measurement_attachment_without_thresholds_when_compiling_then_pol tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (audits [rate]); SELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.', audits [rate]); SELECT 1", "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate); MEASURE (SELECT 100 AS rate FROM @relation); @@ -201,7 +201,7 @@ def test_given_measurement_attachment_with_severity_when_compiling_then_conflict { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (audits [rate (severity warn, thresholds (warn (below 100)))]); SELECT 1" + "MODEL (description 'Test model orders.', audits [rate (severity warn, thresholds (warn (below 100)))]); SELECT 1" ), "audits/generic/rate.sql": """ AUDIT (evaluation measurement, value rate); @@ -234,8 +234,8 @@ def test_given_standalone_measurement_with_header_policy_when_compiling_then_it_ tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\nSELECT 1 AS order_id", - "models/customers.sql": "MODEL ();\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id", + "models/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1 AS order_id", "audits/singular/rate.sql": """ AUDIT ( evaluation measurement, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 504f4cc9d9..8116e86214 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -46,7 +46,8 @@ def test_given_quoted_case_semantics_when_compiling_then_resolves_native_aliases ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql ) main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -77,7 +78,9 @@ def test_given_quoted_typo_when_validating_then_keeps_authored_spelling_and_span 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n' ) (tmp_path / "models").mkdir() - sql: str = "MODEL (database warehouse, schema analytics);\n" + test_case.sql + sql: str = ( + "MODEL (description 'Test model.', database warehouse, schema analytics);\n" + test_case.sql + ) (tmp_path / "models/orders.sql").write_text(sql) assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 payload: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -105,9 +108,12 @@ def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_ ) (tmp_path / "models").mkdir() (tmp_path / "sources").mkdir() - (tmp_path / "sources/orders.yml").write_text("sources:\n - name: orders\n table: orders\n") + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n" + ) (tmp_path / "models/report.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', database warehouse, schema analytics);\n" + + test_case.sql ) project: CompiledProject = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() @@ -164,7 +170,8 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics);\n" + + test_case.sql ) def repeated_binding(**kwargs: Any) -> None: @@ -211,7 +218,7 @@ def test_given_target_cache_policy_when_compiling_twice_then_reuses_only_enabled (tmp_path / "models").mkdir() model: Path = tmp_path / "models/orders.sql" model.write_text( - 'MODEL (database warehouse, schema analytics);\nSELECT SUM("Active") FROM (SELECT TRUE AS "Active") q' + 'MODEL (description "Test model orders.", database warehouse, schema analytics);\nSELECT SUM("Active") FROM (SELECT TRUE AS "Active") q' ) args: list[str] = [ "--project-dir", @@ -261,11 +268,12 @@ def test_given_quoted_catalog_binding_when_compiling_then_preserves_bound_expres ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS amount\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS id, 2 AS amount\n" ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (database warehouse, schema analytics); " + test_case.sql + "MODEL (description 'Test model orders.', database warehouse, schema analytics); " + + test_case.sql ) assert ( main( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index c7e54f315a..6503f5f1e2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -25,19 +25,19 @@ ) _PROJECT_FILES: dict[str, str] = { "sources/raw.yml": ( - "sources:\n - name: raw_orders\n managed: true\n write_strategy: table\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n managed: true\n write_strategy: table\n" " columns:\n - name: order_id\n type: INTEGER\n" ), "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n\n" - "@loader\ndef raw_orders(ctx):\n return [{'order_id': 1}]\n" + "@loader\ndef raw_orders(ctx):\n '''Test loader raw_orders.'''\n return [{'order_id': 1}]\n" ), - "models/order_marker.sql": "MODEL (materialized table);\nSELECT 1 AS n\n", + "models/order_marker.sql": "MODEL (description 'Test model order_marker.', materialized table);\nSELECT 1 AS n\n", "python/tasks/count.py": ( "from sqlbuild.refs import source\nfrom sqlbuild.tasks import task\n\n\n" '@task(depends_on=source("raw_orders"))\n' "def count_task(ctx):\n" - " ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" + " '''Test task count_task.'''\n ctx.query(f\"SELECT count(*) FROM {ctx.relation(source('raw_orders'))}\")\n" ), } diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py index 6a293153d9..bf704a1dc1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py @@ -49,11 +49,13 @@ def test_given_open_sources_when_validating_clauses_then_only_closed_shapes_prov ) (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n" - " - name: known\n table: known\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n - name: customers\n description: Test source customers.\n table: customers\n" + " - name: known\n description: Test source known.\n table: known\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n" ) (tmp_path / "models").mkdir() - (tmp_path / "models/report.sql").write_text("MODEL (materialized view);\n" + test_case.sql) + (tmp_path / "models/report.sql").write_text( + "MODEL (description 'Test model report.', materialized view);\n" + test_case.sql + ) result: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes @@ -102,13 +104,15 @@ def test_given_nested_function_calls_when_compiling_then_native_normalization_do functions: Path = tmp_path / "functions/sql" functions.mkdir(parents=True) (functions / "table_fn__orders.sql").write_text( - "FUNCTION (arguments (x INTEGER), returns table (id INTEGER)); SELECT x AS id" + "FUNCTION (description 'Test function table_fn__orders.', arguments (x INTEGER), returns table (id INTEGER)); SELECT x AS id" ) (functions / "udf__quantity.sql").write_text( - "FUNCTION (arguments (x INTEGER), returns INTEGER); x + 1" + "FUNCTION (description 'Test function udf__quantity.', arguments (x INTEGER), returns INTEGER); x + 1" ) (tmp_path / "models").mkdir() - (tmp_path / "models/orders.sql").write_text("MODEL (materialized view); " + test_case.sql) + (tmp_path / "models/orders.sql").write_text( + "MODEL (description 'Test model orders.', materialized view); " + test_case.sql + ) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes @@ -150,7 +154,7 @@ def without_deferred_requests(**kwargs: Any) -> Any: f'name = "orders"\nadapter = "{test_case.dialect}"\n[rules]\nselect = []\n' ) (tmp_path / "models").mkdir() - header: str = "MODEL (materialized view, database warehouse, schema analytics);\n" + header: str = "MODEL (description 'Test model.', materialized view, database warehouse, schema analytics);\n" (tmp_path / "models/orders.sql").write_text(header + f"SELECT 1 AS {test_case.projection}") (tmp_path / "models/report.sql").write_text( header + f'SELECT {test_case.reference} FROM __ref("orders")' @@ -166,15 +170,15 @@ def without_deferred_requests(**kwargs: Any) -> Any: [ NativeCatalogCase( "accepted numeric strings", - 'MODEL (materialized view, columns (quantity (audits [accepted_values (values ["1", "2"])]))); SELECT 1 AS quantity', + 'MODEL (description "Test model.", materialized view, columns (quantity (audits [accepted_values (values ["1", "2"])]))); SELECT 1 AS quantity', ), NativeCatalogCase( "relationship case", - 'MODEL (materialized view, columns (customer_id (audits [relationships (to __ref("customers"), field customer_id)]))); SELECT 1 AS customer_id', + 'MODEL (description "Test model.", materialized view, columns (customer_id (audits [relationships (to __ref("customers"), field customer_id)]))); SELECT 1 AS customer_id', ), NativeCatalogCase( "relationship coercion", - "MODEL (materialized view, columns (customer_id (audits [relationships (to __ref(\"customers\"), field customer_id)]))); SELECT '1' AS customer_id", + "MODEL (description 'Test model.', materialized view, columns (customer_id (audits [relationships (to __ref(\"customers\"), field customer_id)]))); SELECT '1' AS customer_id", ), ], ids=lambda case: case.description, @@ -190,7 +194,7 @@ def test_given_metadata_predicates_when_compiling_then_allows_dialect_coercion_a (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text(test_case.sql) (tmp_path / "models/customers.sql").write_text( - "MODEL (materialized view); SELECT 1 AS CUSTOMER_ID" + "MODEL (description 'Test model customers.', materialized view); SELECT 1 AS CUSTOMER_ID" ) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py index 550ec61f2c..9f6331740c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py @@ -20,13 +20,13 @@ "sqlbuild_project.toml": _PROJECT_TOML, "sources/raw.yml": ( "sources:\n" - " - name: raw_payments\n" + " - name: raw_payments\n description: Test source raw_payments.\n" " expression: SELECT 1 AS payment_id\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id\n" ), "models/stg_payments.sql": ( - "MODEL (materialized table, audits [orders_backed]);\n\n" + "MODEL (description 'Test model stg_payments.', materialized table, audits [orders_backed]);\n\n" 'SELECT payment_id FROM __source("raw_payments")' ), "audits/generic/orders_backed.sql": ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py index 034fd936a3..809ffa7432 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py @@ -16,7 +16,7 @@ ) from tests.integration.src.sqlbuild.compiler.pipeline.helpers import write_semantic_binding_project -_AUTHORITATIVE_MODEL: str = """MODEL ( +_AUTHORITATIVE_MODEL: str = """MODEL (description "Test model.", materialized table contract enforced columns ( @@ -114,7 +114,7 @@ def test_given_complete_contract_when_clause_references_missing_column_then_comp write_semantic_binding_project( project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, - downstream_sql=f"MODEL (materialized view);\n{test_case.query_sql}\n", + downstream_sql=f"MODEL (description 'Test model.', materialized view);\n{test_case.query_sql}\n", ) exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile", "--no-cache"]) @@ -150,7 +150,7 @@ def test_given_two_complete_contracts_when_unqualified_column_is_ambiguous_then_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("upstream") l CROSS JOIN __ref("other") r\n' ), ) @@ -184,9 +184,9 @@ def test_given_explicit_projection_when_output_column_is_missing_then_compile_fa ) -> None: write_semantic_binding_project( project_dir=tmp_path, - upstream_sql="MODEL (materialized table);\nSELECT id FROM raw_orders\n", + upstream_sql="MODEL (description 'Test model.', materialized table);\nSELECT id FROM raw_orders\n", downstream_sql=( - 'MODEL (materialized view);\nSELECT missing_column FROM __ref("upstream")\n' + 'MODEL (description "Test model.", materialized view);\nSELECT missing_column FROM __ref("upstream")\n' ), ) @@ -233,6 +233,7 @@ def test_given_source_contract_default_when_compiling_then_effective_policy_cont _ = (sources_dir / "raw.yml").write_text( f"""sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders {test_case.source_contract_yaml} columns: @@ -242,7 +243,7 @@ def test_given_source_contract_default_when_compiling_then_effective_policy_cont encoding="utf-8", ) _ = (models_dir / "downstream.sql").write_text( - "MODEL (materialized view, columns (missing (type INTEGER)));\n" + "MODEL (description 'Test model downstream.', materialized view, columns (missing (type INTEGER)));\n" 'SELECT CAST(missing AS INTEGER) AS missing FROM __source("raw_orders")\n', encoding="utf-8", ) @@ -274,12 +275,12 @@ def test_given_complete_inferred_intermediate_when_downstream_column_is_missing_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("intermediate") WHERE missing_derived = TRUE\n' ), ) _ = (tmp_path / "models" / "intermediate.sql").write_text( - 'MODEL (materialized view);\nSELECT id FROM __ref("upstream")\n', + 'MODEL (description "Test model intermediate.", materialized view);\nSELECT id FROM __ref("upstream")\n', encoding="utf-8", ) @@ -309,7 +310,7 @@ def test_given_model_analysis_disabled_when_column_is_missing_then_binding_is_sk project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " sql_analysis false\n" " contract enforced\n" @@ -349,12 +350,12 @@ def test_given_cached_dependent_error_when_upstream_analysis_disabled_then_stale project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - 'MODEL (materialized view);\nSELECT runtime_column FROM __ref("intermediate")\n' + 'MODEL (description "Test model.", materialized view);\nSELECT runtime_column FROM __ref("intermediate")\n' ), ) intermediate_path: Path = tmp_path / "models" / "intermediate.sql" _ = intermediate_path.write_text( - 'MODEL (materialized view);\nSELECT id FROM __ref("upstream")\n', + 'MODEL (description "Test model intermediate.", materialized view);\nSELECT id FROM __ref("upstream")\n', encoding="utf-8", ) @@ -364,7 +365,7 @@ def test_given_cached_dependent_error_when_upstream_analysis_disabled_then_stale assert "error[B002]: Unknown column 'runtime_column'" in initial_output _ = intermediate_path.write_text( - "MODEL (materialized view, sql_analysis false);\n" + "MODEL (description 'Test model.', materialized view, sql_analysis false);\n" 'SELECT runtime_column FROM __ref("upstream")\n', encoding="utf-8", ) @@ -401,7 +402,7 @@ def test_given_cli_analysis_disabled_when_sql_is_unsupported_then_compile_succee models_dir: Path = tmp_path / "models" models_dir.mkdir() _ = (models_dir / "vendor.sql").write_text( - "MODEL (materialized view);\nSELEC unsupported vendor sql\n", + "MODEL (description 'Test model vendor.', materialized view);\nSELEC unsupported vendor sql\n", encoding="utf-8", ) @@ -438,7 +439,7 @@ def test_given_lineage_uses_requested_when_query_filters_then_json_keeps_predica project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" 'SELECT id FROM __ref("upstream") u ' "WHERE u.category = 'active' ORDER BY u.id\n" ), @@ -491,7 +492,7 @@ def test_given_valid_using_and_qualify_alias_when_binding_then_compile_succeeds( project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (materialized view);\n" + "MODEL (description 'Test model.', materialized view);\n" "SELECT ROW_NUMBER() OVER (ORDER BY l.id) AS rn " 'FROM __ref("upstream") l JOIN __ref("upstream") r USING (id) ' "QUALIFY rn = 1\n" @@ -524,7 +525,7 @@ def test_given_unresolved_star_when_contract_has_additional_column_then_compile_ project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " contract enforced\n" " columns (\n" @@ -563,7 +564,7 @@ def test_given_sized_varchar_cast_when_compiling_enforced_contract_then_type_mat project_dir=tmp_path, upstream_sql=_AUTHORITATIVE_MODEL, downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " contract enforced\n" " columns (category (type VARCHAR(3)))\n" @@ -608,6 +609,7 @@ def test_given_untyped_source_when_cast_flows_through_ctes_then_declared_type_ma _ = (sources_dir / "raw.yml").write_text( """sources: - name: raw_orders + description: Test source raw_orders. schema: raw table: orders columns: @@ -616,7 +618,7 @@ def test_given_untyped_source_when_cast_flows_through_ctes_then_declared_type_ma encoding="utf-8", ) model_path: Path = models_dir / "orders.sql" - model_sql: str = """MODEL ( + model_sql: str = """MODEL (description "Test model orders.", materialized view database analytics schema analytics @@ -693,7 +695,7 @@ def test_given_typed_union_ctes_when_compiling_enforced_contract_then_types_are_ " materialized table\n database analytics\n schema analytics\n", ), downstream_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view\n" " database analytics\n" " schema analytics\n" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py index ab772d325c..f54b291d9a 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -69,7 +69,7 @@ extra_files=( ( "models/independent.sql", - 'MODEL (materialized view);\nSELECT w.missing FROM __seed("waffle_types") w CROSS JOIN __ref("fact_orders") o\n', + 'MODEL (description "Test model independent.", materialized view);\nSELECT w.missing FROM __seed("waffle_types") w CROSS JOIN __ref("fact_orders") o\n', ), ), expected_errors=2, @@ -88,14 +88,17 @@ "predicate error does not poison another input projection", (), extra_files=( - ("models/available.sql", "MODEL (materialized view); SELECT 1 AS qty"), + ( + "models/available.sql", + "MODEL (description 'Test model available.', materialized view); SELECT 1 AS qty", + ), ( "models/predicate_bad.sql", - 'MODEL (materialized view); SELECT a.qty FROM __ref("available") a CROSS JOIN __ref("stg_orders") o WHERE o.qty > 5', + 'MODEL (description "Test model predicate_bad.", materialized view); SELECT a.qty FROM __ref("available") a CROSS JOIN __ref("stg_orders") o WHERE o.qty > 5', ), ( "models/independent_use.sql", - 'MODEL (materialized view); SELECT quantity FROM __ref("predicate_bad")', + 'MODEL (description "Test model independent_use.", materialized view); SELECT quantity FROM __ref("predicate_bad")', ), ), expected_errors=2, @@ -110,11 +113,11 @@ extra_files=( ( "sources/optional.yml", - "sources:\n - name: external_orders\n table: external_orders\n", + "sources:\n - name: external_orders\n description: Test source external_orders.\n table: external_orders\n", ), ( "models/open_read.sql", - 'MODEL (materialized view);\nSELECT id FROM __source("external_orders")\n', + 'MODEL (description "Test model open_read.", materialized view);\nSELECT id FROM __source("external_orders")\n', ), ), expected_fragments=( @@ -139,12 +142,12 @@ extra_files=( ( "models/wide.sql", - "MODEL (materialized view);\nSELECT 1 AS quantity, " + "MODEL (description 'Test model wide.', materialized view);\nSELECT 1 AS quantity, " + ", ".join(f"1 AS field_{index}" for index in range(20)), ), ( "models/wide_bad.sql", - 'MODEL (materialized view);\nSELECT w.qty FROM __ref("wide") w', + 'MODEL (description "Test model wide_bad.", materialized view);\nSELECT w.qty FROM __ref("wide") w', ), ), expected_fragments=("wide has: quantity", "and 11 more", "did you mean 'quantity'?"), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py index 9641558072..96e61e1311 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py @@ -75,7 +75,7 @@ _UPSTREAM, "SELECT 1 AS id", "B300", - "MODEL (materialized table, unique_key [missing]);\n", + "MODEL (description 'Test model.', materialized table, unique_key [missing]);\n", ), SemanticCompileCase( "date literal valid control", @@ -117,16 +117,16 @@ def test_given_semantic_scope_when_compiling_then_closed_shapes_are_checked( ) -> None: write_semantic_binding_project( project_dir=tmp_path, - upstream_sql="MODEL (materialized view);\n" + test_case.upstream, + upstream_sql="MODEL (description 'Test model.', materialized view);\n" + test_case.upstream, downstream_sql=test_case.header + test_case.downstream, ) (tmp_path / "sources").mkdir(exist_ok=True) (tmp_path / "sources" / "orders.yml").write_text( "sources:\n" - " - name: open_orders\n table: orders\n" - " - name: closed_orders\n table: orders\n contract: enforced\n" + " - name: open_orders\n description: Test source open_orders.\n table: orders\n" + " - name: closed_orders\n description: Test source closed_orders.\n table: orders\n contract: enforced\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: expression_orders\n expression: SELECT 1 AS id\n", + " - name: expression_orders\n description: Test source expression_orders.\n expression: SELECT 1 AS id\n", encoding="utf-8", ) exit_code: int = main( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index 342e9704bd..ff4cb8db7c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -91,7 +91,7 @@ def test_given_snowflake_semantics_when_compiling_then_preserves_native_verdicts (tmp_path / "models").mkdir() (tmp_path / "sources").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n" " - {name: order_id, type: INTEGER}\n" " - {name: quantity, type: INTEGER}\n" " - {name: value, type: VARCHAR}\n" @@ -99,7 +99,8 @@ def test_given_snowflake_semantics_when_compiling_then_preserves_native_verdicts " - {name: ordered_at, type: TIMESTAMP}\n" ) (tmp_path / "models/report.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', materialized view, database warehouse, schema analytics);\n" + + test_case.sql ) assert ( main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) @@ -135,7 +136,8 @@ def test_given_snowflake_query_when_inferring_rich_types_then_preserves_proven_o (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') (tmp_path / "models").mkdir() (tmp_path / "models/report.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + "MODEL (description 'Test model report.', materialized view, database warehouse, schema analytics);\n" + + test_case.sql ) project: CompiledProject = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index b96ee287a2..e0be6124d6 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -44,11 +44,11 @@ def test_given_cursor_families_when_compiling_then_accepts_temporal_pairs_only( (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n" f" columns:\n - name: ordered_at\n type: {test_case.column_type}\n" ) (tmp_path / "models/recent_orders.sql").write_text( - "MODEL (materialized incremental, database warehouse, schema analytics, " + "MODEL (description 'Test model recent_orders.', materialized incremental, database warehouse, schema analytics, " f"incremental_strategy append, cursor ordered_at, cursor_type {test_case.cursor_type}, " 'cursor_grain day);\nSELECT ordered_at FROM __source("orders")' ) @@ -83,11 +83,11 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() (tmp_path / "sources/orders.yml").write_text( - "sources:\n - name: orders\n table: orders\n contract: enforced\n" + "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n" " columns:\n - name: customer\n type: VARCHAR\n" ) (tmp_path / "models/quoted_orders.sql").write_text( - "MODEL (materialized view, database warehouse, schema analytics);\n" + "MODEL (description 'Test model quoted_orders.', materialized view, database warehouse, schema analytics);\n" 'SELECT o."customer" FROM __source("orders") o' ) for _ in range(2): @@ -138,21 +138,21 @@ def test_given_type_error_root_when_binding_dependants_then_recovers_only_poison ) (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), " + "MODEL (description 'Test model orders.', materialized view, contract enforced, columns (quantity (type INTEGER), " "ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, " "CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" ) (tmp_path / "models/broken.sql").write_text( - f"MODEL (materialized view{test_case.root_contract});\nSELECT quantity + ordered_at AS total, " + f"MODEL (description 'Test model broken.', materialized view{test_case.root_contract});\nSELECT quantity + ordered_at AS total, " 'ordered_at FROM __ref("orders")' ) (tmp_path / "models/downstream.sql").write_text( - "MODEL (materialized view);\nSELECT total > ordered_at AS compared" + "MODEL (description 'Test model downstream.', materialized view);\nSELECT total > ordered_at AS compared" + test_case.independent_sql + ' FROM __ref("broken")' ) (tmp_path / "models/final_orders.sql").write_text( - 'MODEL (materialized view);\nSELECT compared AS final_value FROM __ref("downstream")' + 'MODEL (description "Test model final_orders.", materialized view);\nSELECT compared AS final_value FROM __ref("downstream")' ) main(["--project-dir", str(tmp_path), "compile", "--json"]) result: dict[str, Any] = json.loads(capsys.readouterr().out) @@ -181,11 +181,11 @@ def test_given_repetitive_sql_when_mapping_diagnostic_then_preserves_authored_sp (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') (tmp_path / "models").mkdir() (tmp_path / "models/orders.sql").write_text( - "MODEL (materialized view);\nSELECT CAST(1 AS INTEGER) AS quantity" + "MODEL (description 'Test model orders.', materialized view);\nSELECT CAST(1 AS INTEGER) AS quantity" ) projections: str = ",\n".join(f"quantity + {index} AS amount_{index}" for index in range(300)) sql: str = ( - "MODEL (materialized view);\nSELECT\n" + "MODEL (description 'Test model s.', materialized view);\nSELECT\n" + projections + "\nFROM __ref(\"orders\")\nWHERE quantity > TIMESTAMP '2026-04-01'" ) @@ -216,10 +216,10 @@ def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_ma (tmp_path / "sources").mkdir() (tmp_path / "models").mkdir() source: Path = tmp_path / "sources/orders.yml" - declaration: str = "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n - name: quantity\n type: INTEGER\n" + declaration: str = "sources:\n - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n - name: quantity\n type: INTEGER\n" source.write_text(declaration) (tmp_path / "models/totals.sql").write_text( - 'MODEL (materialized view);\nSELECT quantity + 1 AS total FROM __source("orders")' + 'MODEL (description "Test model totals.", materialized view);\nSELECT quantity + 1 AS total FROM __source("orders")' ) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] with monkeypatch.context() as patch: @@ -258,10 +258,10 @@ def test_given_repeated_root_messages_when_recovering_then_preserves_each_output ) (tmp_path / "models").mkdir() files: dict[str, str] = { - "orders": "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", - "broken": 'MODEL (materialized view, contract enforced, columns (total_a (type INTEGER), total_b (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT quantity + ordered_at AS total_a, quantity + ordered_at AS total_b, ordered_at FROM __ref("orders")', - "downstream": 'MODEL (materialized view);\nSELECT total_a > ordered_at AS a, total_b > ordered_at AS b FROM __ref("broken")', - "final_orders": 'MODEL (materialized view);\nSELECT a, b FROM __ref("downstream")', + "orders": "MODEL (description 'Test model.', materialized view, contract enforced, columns (quantity (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", + "broken": 'MODEL (description "Test model.", materialized view, contract enforced, columns (total_a (type INTEGER), total_b (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT quantity + ordered_at AS total_a, quantity + ordered_at AS total_b, ordered_at FROM __ref("orders")', + "downstream": 'MODEL (description "Test model.", materialized view);\nSELECT total_a > ordered_at AS a, total_b > ordered_at AS b FROM __ref("broken")', + "final_orders": 'MODEL (description "Test model.", materialized view);\nSELECT a, b FROM __ref("downstream")', } for name, sql in files.items(): (tmp_path / "models" / f"{name}.sql").write_text(sql) @@ -322,20 +322,20 @@ def test_given_udf_argument_columns_when_compiling_then_checks_declared_argument (tmp_path / "functions/sql").mkdir(parents=True) (tmp_path / "sources/raw.yml").write_text( "sources:\n" - " - name: orders\n table: orders\n contract: enforced\n columns:\n" + " - name: orders\n description: Test source orders.\n table: orders\n contract: enforced\n columns:\n" " - name: id\n type: INTEGER\n" " - name: customer_id\n type: INTEGER\n" " - name: status\n type: VARCHAR\n" " - name: amount\n type: DOUBLE\n" - " - name: customers\n table: customers\n contract: enforced\n columns:\n" + " - name: customers\n description: Test source customers.\n table: customers\n contract: enforced\n columns:\n" " - name: id\n type: INTEGER\n" " - name: customer_name\n type: VARCHAR\n" ) (tmp_path / "functions/sql/add_one.sql").write_text( - "FUNCTION (\n arguments (input_value DOUBLE),\n returns DOUBLE,\n);\n\ninput_value + 1\n" + "FUNCTION (description 'Test function add_one.',\n arguments (input_value DOUBLE),\n returns DOUBLE,\n);\n\ninput_value + 1\n" ) (tmp_path / "models/order_totals.sql").write_text( - f"MODEL (materialized table);\n\nSELECT {test_case.projection}\n" + f"MODEL (description 'Test model order_totals.', materialized table);\n\nSELECT {test_case.projection}\n" 'FROM __source("orders") o JOIN __source("customers") c ON o.customer_id = c.id\n' ) main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index b2b49e39e4..c794065571 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -57,7 +57,8 @@ def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - f"MODEL (materialized table);\n{test_case.query_sql}", encoding="utf-8" + f"MODEL (description 'Test model orders.', materialized table);\n{test_case.query_sql}", + encoding="utf-8", ) project: CompiledProject = compile_project( @@ -95,19 +96,19 @@ def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model orders.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" "SELECT CAST(2.5 AS DOUBLE) AS amount", encoding="utf-8", ) (models / "combined_orders.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model combined_orders.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" 'WITH combined AS (SELECT CAST(amount AS DOUBLE) AS amount FROM __ref("orders") ' 'UNION ALL SELECT list_extract([amount], 1) AS amount FROM __ref("refunds")) ' "SELECT amount FROM combined", encoding="utf-8", ) (models / "refunds.sql").write_text( - "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "MODEL (description 'Test model refunds.', materialized table, contract enforced, columns (amount (type DOUBLE)));\n" "SELECT CAST(1.25 AS DOUBLE) AS amount", encoding="utf-8", ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py index 08d36c33c2..cd1d1b2cd1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py @@ -20,53 +20,53 @@ [ SharedBindingQueryCase( description="identical shapes and inputs share one query", - orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', - customers_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("customers")', + orders_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', + customers_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers")', expected_shared_queries=1, ), SharedBindingQueryCase( description="a relation name used as a qualifier is not shared", orders_summary_sql=( - 'MODEL (materialized view); SELECT orders.id FROM __ref("orders") AS orders' + 'MODEL (description "Test model.", materialized view); SELECT orders.id FROM __ref("orders") AS orders' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT customers.id FROM __ref("customers") AS customers' + 'MODEL (description "Test model.", materialized view); SELECT customers.id FROM __ref("customers") AS customers' ), expected_shared_queries=0, ), SharedBindingQueryCase( description="an authored lowercase stub-named CTE is not shared", orders_summary_sql=( - "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __sqlbuild_project_input_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' ), customers_summary_sql=( - "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __sqlbuild_project_input_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' ), expected_shared_queries=0, later_models=( ( "next_orders", - 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', ), ), ), SharedBindingQueryCase( description="an authored uppercase stub-named CTE is not shared", orders_summary_sql=( - "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' ), customers_summary_sql=( - "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + "MODEL (description 'Test model.', materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' ), expected_shared_queries=0, later_models=( ( "next_orders", - 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', ), ), ), @@ -116,10 +116,10 @@ def test_given_models_with_equal_shapes_when_compiling_then_shares_only_lossless SharedBindingQueryCase( description="findings after the relation keep each model's names and positions", orders_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' ), expected_shared_queries=1, expected_codes=("B002",), @@ -163,19 +163,19 @@ def test_given_shared_query_with_finding_when_compiling_then_reports_each_model_ [ SharedBindingQueryCase( description="later-wave members reuse an exact shared result", - orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', + orders_summary_sql='MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', customers_summary_sql=( - 'MODEL (materialized view); SELECT id, amount FROM __ref("customers")' + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers")' ), expected_shared_queries=1, later_models=( ( "orders_rollup", - 'MODEL (materialized view); SELECT id, amount FROM __ref("orders_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders_summary")', ), ( "customers_rollup", - 'MODEL (materialized view); SELECT id, amount FROM __ref("customers_summary")', + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("customers_summary")', ), ), expected_lineage=( @@ -235,18 +235,21 @@ def test_given_later_wave_members_with_equal_keys_when_compiling_then_reuses_sha SharedBindingQueryCase( description="a later-wave member of a group with findings keeps its own names", orders_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' ), customers_summary_sql=( - 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' ), expected_shared_queries=2, expected_codes=("B002",), later_models=( - ("orders_mid", 'MODEL (materialized view); SELECT id, amount FROM __ref("orders")'), + ( + "orders_mid", + 'MODEL (description "Test model.", materialized view); SELECT id, amount FROM __ref("orders")', + ), ( "orders_late", - 'MODEL (materialized view); SELECT id FROM __ref("orders_mid") WHERE missing > 0', + 'MODEL (description "Test model.", materialized view); SELECT id FROM __ref("orders_mid") WHERE missing > 0', ), ), expected_findings=( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index 9ba7d3c23c..3d3b9ece48 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -30,10 +30,10 @@ def test_given_expanded_mixed_mode_tests_when_compiling_project_then_complete_in tmp_path, { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", "models/_constants/order_id.sql": "CONSTANT (name order_id, value 7);\n", "functions/sql/increment.sql": ( - "FUNCTION (arguments (value INTEGER), returns INTEGER);\n\nvalue + 1\n" + "FUNCTION (description 'Test function increment.', arguments (value INTEGER), returns INTEGER);\n\nvalue + 1\n" ), "tests/unit/a_orders.sql": ( "TEST (name orders_match);\n\n" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 8d0cee809f..dfbfa47666 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -59,9 +59,11 @@ def test_given_parameterized_model_test_when_compiled_and_executed_then_duckdb_p { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "sources/raw.yml": ( - "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT order_id FROM __source("raw_orders")\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT order_id FROM __source("raw_orders")\n'), "tests/unit/orders.sql": f""" TEST ( name "parameterized_orders", diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 2542323465..82d1a51f31 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -42,11 +42,11 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir 'database = ":memory:"\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT order_id, status FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.");\n\nSELECT order_id, status FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT 1 AS order_id, 'open' AS status\n" " columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" @@ -109,12 +109,12 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ 'database = ":memory:"\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT TRY_CAST(status AS DOUBLE) AS parsed_status " + "MODEL (description 'Test model orders.');\n\nSELECT TRY_CAST(status AS DOUBLE) AS parsed_status " 'FROM __source("raw_orders")\n' ), "sources/raw_orders.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " expression: SELECT '1.5' AS status\n" " contract: enforced\n" " columns:\n" diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 5d4b27c89f..7c7fde33f7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -595,11 +595,13 @@ def plan_source_reading_project( project_files: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "sources/raw.yml": ( - f"sources:\n - name: raw_orders\n schema: {source_schema}\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + f" schema: {source_schema}\n" f" table: {source_table}\n" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), } relative_path: str diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py index c07fce1696..8d1810466c 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py @@ -39,10 +39,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/alpha.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alpha' AS name" + "MODEL (description 'Test model alpha.', materialized table);\n\nSELECT 1 AS id, 'alpha' AS name" ), "models/beta.sql": ( - "MODEL (materialized table);\n\nSELECT 2 AS id, 'beta' AS name" + "MODEL (description 'Test model beta.', materialized table);\n\nSELECT 2 AS id, 'beta' AS name" ), }, expected_status=BuildStatus.SUCCESS, @@ -64,18 +64,18 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/staging/stg_raw.sql": ( - "MODEL (materialized view);\n\nSELECT 10 AS id, 'row' AS val" + "MODEL (description 'Test model stg_raw.', materialized view);\n\nSELECT 10 AS id, 'row' AS val" ), "models/branch_a.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model branch_a.', materialized table);\n\n" "SELECT id, val || '_a' AS val FROM __ref(\"stg_raw\")" ), "models/branch_b.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model branch_b.', materialized table);\n\n" "SELECT id, val || '_b' AS val FROM __ref(\"stg_raw\")" ), "models/merged.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model merged.', materialized table);\n\n" "SELECT a.id, a.val AS a_val, b.val AS b_val " 'FROM __ref("branch_a") a ' 'JOIN __ref("branch_b") b ON a.id = b.id' @@ -95,12 +95,14 @@ max_concurrency=2, project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/layer_1.sql": ("MODEL (materialized table);\n\nSELECT 100 AS val"), + "models/layer_1.sql": ( + "MODEL (description 'Test model layer_1.', materialized table);\n\nSELECT 100 AS val" + ), "models/layer_2.sql": ( - 'MODEL (materialized table);\n\nSELECT val * 2 AS val FROM __ref("layer_1")' + 'MODEL (description "Test model layer_2.", materialized table);\n\nSELECT val * 2 AS val FROM __ref("layer_1")' ), "models/layer_3.sql": ( - 'MODEL (materialized table);\n\nSELECT val + 1 AS val FROM __ref("layer_2")' + 'MODEL (description "Test model layer_3.", materialized table);\n\nSELECT val + 1 AS val FROM __ref("layer_2")' ), }, expected_status=BuildStatus.SUCCESS, @@ -170,6 +172,7 @@ def test_given_concurrent_provider_backed_loaders_when_executing_then_share_prov class ConcurrentMarkerProvider(Provider): + '''Test provider ConcurrentMarkerProvider.''' marker_path: str = {str(marker_path)!r} @property @@ -197,12 +200,14 @@ def mark(self, label): @loader def raw_alpha(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_alpha.''' concurrent_marker_provider.mark("alpha") return [{"event_id": 1}] @loader def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): + '''Test loader raw_beta.''' concurrent_marker_provider.mark("beta") return [{"event_id": 2}] """ @@ -212,12 +217,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): """ sources: - name: raw_alpha + description: Test source raw_alpha. managed: true write_strategy: table columns: - name: event_id type: INTEGER - name: raw_beta + description: Test source raw_beta. managed: true write_strategy: table columns: @@ -227,10 +234,10 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): ).strip() + "\n", "models/fact_alpha.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' + 'MODEL (description "Test model fact_alpha.", materialized table);\n\nSELECT * FROM __source("raw_alpha")\n' ), "models/fact_beta.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_beta")\n' + 'MODEL (description "Test model fact_beta.", materialized table);\n\nSELECT * FROM __source("raw_beta")\n' ), }, ) @@ -269,9 +276,11 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/healthy.sql": ( + "MODEL (description 'Test model healthy.', materialized table);\n\nSELECT 42 AS val" ), - "models/healthy.sql": ("MODEL (materialized table);\n\nSELECT 42 AS val"), }, expected_status=BuildStatus.FAILED, expected_success_count=1, @@ -291,12 +300,14 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model broken.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/downstream.sql": ( - 'MODEL (materialized table);\n\nSELECT 1 AS id FROM __ref("broken")' + 'MODEL (description "Test model downstream.", materialized table);\n\nSELECT 1 AS id FROM __ref("broken")' + ), + "models/independent.sql": ( + "MODEL (description 'Test model independent.', materialized table);\n\nSELECT 99 AS val" ), - "models/independent.sql": ("MODEL (materialized table);\n\nSELECT 99 AS val"), }, expected_status=BuildStatus.FAILED, expected_success_count=1, @@ -319,9 +330,11 @@ def raw_beta(ctx, concurrent_marker_provider: ConcurrentMarkerProvider): project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/aaa_broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model aaa_broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/zzz_independent.sql": ( + "MODEL (description 'Test model zzz_independent.', materialized table);\n\nSELECT 1 AS id" ), - "models/zzz_independent.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), }, fail_fast=True, expected_status=BuildStatus.FAILED, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 98193b4a7e..272150769b 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -118,6 +118,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -125,7 +126,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( + "\n", "models/orders.sql": dedent( """ - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -150,7 +151,7 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( + "\n", "models/shipments.sql": dedent( """ - MODEL ( + MODEL (description "Test model shipments.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 897b8acf68..9785765a12 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -52,16 +52,20 @@ max_concurrency=3, project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/staging/stg_a.sql": ("MODEL (materialized view);\n\nSELECT 1 AS id"), - "models/staging/stg_b.sql": ("MODEL (materialized view);\n\nSELECT 2 AS id"), + "models/staging/stg_a.sql": ( + "MODEL (description 'Test model stg_a.', materialized view);\n\nSELECT 1 AS id" + ), + "models/staging/stg_b.sql": ( + "MODEL (description 'Test model stg_b.', materialized view);\n\nSELECT 2 AS id" + ), "models/mid_a.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_a")' + 'MODEL (description "Test model mid_a.", materialized table);\n\nSELECT id FROM __ref("stg_a")' ), "models/mid_b.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_b")' + 'MODEL (description "Test model mid_b.", materialized table);\n\nSELECT id FROM __ref("stg_b")' ), "models/final.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model final.', materialized table);\n\n" "SELECT a.id AS a_id, b.id AS b_id " 'FROM __ref("mid_a") a ' 'CROSS JOIN __ref("mid_b") b' diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py index 18b9028045..83944e2890 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_stateless_sequential_microbatch.py @@ -78,6 +78,7 @@ def test_given_sequential_microbatch_when_repeated_then_uses_only_physical_water """ sources: - name: raw_events + description: Test source raw_events. schema: main table: raw_events """ @@ -85,7 +86,7 @@ def test_given_sequential_microbatch_when_repeated_then_uses_only_physical_water + "\n", "models/orders.sql": dedent( f""" - MODEL ( + MODEL (description "Test model orders.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py index 22ed209e90..06f7212128 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py @@ -302,10 +302,10 @@ def test_given_custom_materialization_when_audit_finds_rows_in_staging_then_bloc project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/regular.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'regular' AS name" + "MODEL (description 'Test model regular.', materialized table);\n\nSELECT 1 AS id, 'regular' AS name" ), "models/custom_model.sql": ( - "MODEL (materialized test_custom);\n\nSELECT 2 AS id, 'custom' AS name" + "MODEL (description 'Test model custom_model.', materialized test_custom);\n\nSELECT 2 AS id, 'custom' AS name" ), }, expected_status=BuildStatus.SUCCESS, @@ -323,10 +323,10 @@ def test_given_custom_materialization_when_audit_finds_rows_in_staging_then_bloc project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/upstream.sql": ( - "MODEL (materialized table);\n\nSELECT 10 AS id, 'upstream' AS origin" + "MODEL (description 'Test model upstream.', materialized table);\n\nSELECT 10 AS id, 'upstream' AS origin" ), "models/downstream_custom.sql": ( - 'MODEL (materialized test_custom);\n\nSELECT id, origin FROM __ref("upstream")' + 'MODEL (description "Test model downstream_custom.", materialized test_custom);\n\nSELECT id, origin FROM __ref("upstream")' ), }, expected_status=BuildStatus.SUCCESS, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 05e2bb5231..e515555b70 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -50,7 +50,7 @@ def capped_dependency_producer_sql(*, action: MicrobatchLimitAction | None) -> s limit_sql: str = limit_blocks[action] return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -81,7 +81,7 @@ def capped_dependency_consumer_sql( return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -108,7 +108,7 @@ def capped_filter_consumer_sql(*, input_name: str = "capped_events") -> str: return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -136,7 +136,7 @@ def plain_capped_consumer_sql(*, materialized: str) -> str: return dedent( f""" - MODEL (materialized {materialized}); + MODEL (description "Test model.", materialized {materialized}); SELECT id, event_time FROM __ref("capped_events") """ ) @@ -147,7 +147,7 @@ def dependency_view_sql(*, upstream_name: str) -> str: return dedent( f""" - MODEL (materialized view); + MODEL (description "Test model.", materialized view); SELECT id, event_time FROM __ref("{upstream_name}") """ ) @@ -177,7 +177,7 @@ def capped_microbatch_intermediary_sql(*, input_role: str) -> str: } return dedent( f""" - MODEL ( + MODEL (description "Test model.", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -215,7 +215,8 @@ def capped_dependency_warehouse_files(*, project_dir: Path) -> dict[str, str]: f"[connection]\ndatabase = {json.dumps(str(database_path))}\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + " schema: main\n table: raw_events\n" ), } diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index e0e158de04..491d2b3e30 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -426,7 +426,7 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: ), "sources/raw.yml": ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " database: analytics\n" " schema: raw\n" " table: orders\n" @@ -439,14 +439,15 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: " type: NUMBER(10, 2)\n" ), "models/marts/orders_table.sql": ( - "MODEL (\n materialized table,\n);\n\n" + "MODEL (description 'Test model orders_table.',\n materialized table,\n);\n\n" 'SELECT order_id, ordered_at, amount FROM __source("raw_orders")\n' ), "models/marts/orders_view.sql": ( - 'MODEL (\n materialized view,\n);\n\nSELECT order_id, amount FROM __ref("orders_table")\n' + 'MODEL (description "Test model orders_view.",' + '\n materialized view,\n);\n\nSELECT order_id, amount FROM __ref("orders_table")\n' ), "models/marts/orders_merge.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders_merge.',\n" " materialized incremental,\n" " incremental_strategy merge,\n" " unique_key [order_id],\n" @@ -459,7 +460,7 @@ def _show_schema_identity(match: re.Match[str]) -> _Identity: 'SELECT order_id, ordered_at, amount FROM __source("raw_orders")\n' ), "models/marts/orders_daily.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders_daily.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " incremental_mode microbatch,\n" diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index c28eab2166..b565a28996 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -54,7 +54,7 @@ _RAW_ORDERS_WITH_NOT_NULL_AUDIT: str = ( "sources:\n" - " - name: raw_orders\n" + " - name: raw_orders\n description: Test source raw_orders.\n" " schema: main\n" " table: raw_orders\n" " columns:\n" @@ -64,10 +64,10 @@ ) _NOT_NULL_AUDIT: str = 'AUDIT ();\n\nSELECT @column FROM __ref("@model") WHERE @column IS NULL' _TABLE_WITH_ID_NOT_NULL_AUDIT: str = ( - "MODEL (materialized table, columns (id (audits [not_null])));\n\n" + "MODEL (description 'Test model.', materialized table, columns (id (audits [not_null])));\n\n" ) _VIEW_WITH_ID_NOT_NULL_AUDIT: str = ( - "MODEL (materialized view, columns (id (audits [not_null])));\n\n" + "MODEL (description 'Test model.', materialized view, columns (id (audits [not_null])));\n\n" ) @@ -104,14 +104,14 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "functions/sql/is_positive_int.sql": ( - "FUNCTION (\n" + "FUNCTION (description 'Test function is_positive_int.',\n" " arguments (a_string VARCHAR),\n" " returns BOOLEAN\n" ");\n\n" "regexp_matches(a_string, '^[0-9]+$')" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -143,10 +143,10 @@ " runtime_version='3.11',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -181,10 +181,10 @@ " runtime_version='3.11',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' "FROM (VALUES ('123'), ('abc')) AS input(value)" ), @@ -209,9 +209,9 @@ description="two independent models both succeed", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name", "models/payments.sql": ( - "MODEL (materialized table);\n\nSELECT 10 AS payment_id, 500 AS amount" + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 10 AS payment_id, 500 AS amount" ), }, expected_status=BuildStatus.SUCCESS, @@ -230,10 +230,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 42 AS id, 'bob' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 42 AS id, 'bob' AS name" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" 'SELECT id, name FROM __ref("stg_orders") WHERE id = 42' ), }, @@ -261,7 +261,7 @@ 'schema = "dev_schema"\n' ), "sqlbuild_local.toml": 'target = "dev"\n', - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name", }, expected_status=BuildStatus.SUCCESS, expected_success_count=1, @@ -309,7 +309,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (id (audits [unique])));\n\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" "SELECT * FROM (VALUES (1), (NULL), (NULL)) AS input(id)" ), }, @@ -325,7 +325,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (status (audits [accepted_values " + "MODEL (description 'Test model orders.', materialized table, columns (status (audits [accepted_values " '(values ["placed", "completed"])])));\n\n' "SELECT * FROM (VALUES ('placed'), ('completed'), (NULL)) AS input(status)" ), @@ -344,9 +344,9 @@ description="built-in relationships audit allows matching referenced values", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/customers.sql": "MODEL (description 'Test model customers.', materialized table);\n\nSELECT 1 AS id", "models/orders.sql": ( - "MODEL (materialized table, columns (customer_id (audits [relationships " + "MODEL (description 'Test model orders.', materialized table, columns (customer_id (audits [relationships " '(to __ref("customers"), field id)])));\n\n' 'SELECT id AS customer_id FROM __ref("customers") UNION ALL SELECT NULL' ), @@ -379,7 +379,7 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML_WARN, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -394,8 +394,12 @@ description="end audit warn succeeds build with warning", project_files={ "sqlbuild_project.toml": _PROJECT_YML_WARN, - "models/orders.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 2 AS payment_id"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id" + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 2 AS payment_id" + ), "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' ), @@ -414,9 +418,11 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 1 AS payment_id" ), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 1 AS payment_id"), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' @@ -459,7 +465,7 @@ description="build writes fingerprints when query tracking is enabled", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS id", }, expected_status=BuildStatus.SUCCESS, expected_success_count=1, @@ -516,10 +522,10 @@ def test_given_build_plan_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized view);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS id, 'alice' AS name" ), "models/dim_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' + 'MODEL (description "Test model dim_orders.", materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.SUCCESS, @@ -573,10 +579,10 @@ def test_given_view_build_plan_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/bad_view.sql": ( - "MODEL (materialized view);\n\nSELECT * FROM nonexistent_source_table" + "MODEL (description 'Test model bad_view.', materialized view);\n\nSELECT * FROM nonexistent_source_table" ), "models/downstream.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __ref("bad_view")' + 'MODEL (description "Test model downstream.", materialized table);\n\nSELECT * FROM __ref("bad_view")' ), }, expected_status=BuildStatus.FAILED, @@ -656,10 +662,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -677,7 +683,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (id (type INTEGER)),\n" @@ -700,7 +706,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML_IMMEDIATE, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized table,\n" " contract enforced,\n" " columns (id (type INTEGER)),\n" @@ -720,7 +726,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -752,7 +758,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " contract enforced,\n" " columns (\n" @@ -799,10 +805,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( " schema='udfs',\n" ")\n" "def main(a_string):\n" - " return bool(a_string and a_string.isdigit())\n" + " '''Test function main.'''\n return bool(a_string and a_string.isdigit())\n" ), "models/validated_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model validated_orders.', materialized table);\n\n" 'SELECT __udf("is_positive_int")(' "'123'" ") AS is_positive" @@ -827,13 +833,13 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT * FROM nonexistent_table" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), "models/payments.sql": ( - "MODEL (materialized table);\n\nSELECT 99 AS payment_id, 750 AS amount" + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 99 AS payment_id, 750 AS amount" ), }, expected_status=BuildStatus.FAILED, @@ -868,7 +874,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (id (audits [unique])));\n\n" + "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" "SELECT * FROM (VALUES (1), (1), (NULL)) AS input(id)" ), }, @@ -883,7 +889,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table, columns (status (audits [accepted_values " + "MODEL (description 'Test model orders.', materialized table, columns (status (audits [accepted_values " '(values ["placed", "completed"])])));\n\n' "SELECT 'cancelled' AS status" ), @@ -898,9 +904,9 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( description="built-in relationships audit blocks missing referenced values", project_files={ "sqlbuild_project.toml": _PROJECT_YML, - "models/customers.sql": "MODEL (materialized table);\n\nSELECT 1 AS id", + "models/customers.sql": "MODEL (description 'Test model customers.', materialized table);\n\nSELECT 1 AS id", "models/orders.sql": ( - "MODEL (materialized table, columns (customer_id (audits [relationships " + "MODEL (description 'Test model orders.', materialized table, columns (customer_id (audits [relationships " '(to __ref("customers"), field id)])));\n\n' 'SELECT 2 AS customer_id FROM __ref("customers")' ), @@ -920,9 +926,11 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - "MODEL (materialized table);\n\nSELECT 7 AS id, 'carol' AS name" + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 7 AS id, 'carol' AS name" + ), + "models/payments.sql": ( + "MODEL (description 'Test model payments.', materialized table);\n\nSELECT 1 AS payment_id" ), - "models/payments.sql": ("MODEL (materialized table);\n\nSELECT 1 AS payment_id"), "audits/singular/cross_check.sql": ( 'AUDIT ();\n\nSELECT o.id FROM __ref("orders") o CROSS JOIN __ref("payments") p' ), @@ -940,9 +948,11 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/aaa_broken.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_table" + "MODEL (description 'Test model aaa_broken.', materialized table);\n\nSELECT * FROM nonexistent_table" + ), + "models/zzz_healthy.sql": ( + "MODEL (description 'Test model zzz_healthy.', materialized table);\n\nSELECT 1 AS id" ), - "models/zzz_healthy.sql": ("MODEL (materialized table);\n\nSELECT 1 AS id"), }, fail_fast=True, expected_status=BuildStatus.FAILED, @@ -956,7 +966,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -973,10 +983,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")' + 'MODEL (description "Test model stg_orders.", materialized table);\n\nSELECT id FROM __source("raw_orders")' ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), "sources/raw.yml": _RAW_ORDERS_WITH_NOT_NULL_AUDIT, }, @@ -1009,12 +1019,12 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (\n materialized table\n" + "MODEL (description 'Test model stg_orders.',\n materialized table\n" " pre_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 1 AS id" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -1031,12 +1041,12 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (\n materialized table\n" + "MODEL (description 'Test model stg_orders.',\n materialized table\n" " post_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 88 AS id" ), "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")' + 'MODEL (description "Test model orders.", materialized table);\n\nSELECT id FROM __ref("stg_orders")' ), }, expected_status=BuildStatus.FAILED, @@ -1054,10 +1064,10 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/broken_a.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_a" + "MODEL (description 'Test model broken_a.', materialized table);\n\nSELECT * FROM nonexistent_a" ), "models/broken_b.sql": ( - "MODEL (materialized table);\n\nSELECT * FROM nonexistent_b" + "MODEL (description 'Test model broken_b.', materialized table);\n\nSELECT * FROM nonexistent_b" ), }, expected_status=BuildStatus.FAILED, @@ -1115,7 +1125,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _PASSING_TEST_SQL, }, @@ -1130,7 +1140,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, @@ -1147,7 +1157,7 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, @@ -1163,10 +1173,10 @@ def test_given_build_plan_when_executing_then_fails( project_files={ "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( - "MODEL (materialized table);\n\nSELECT 1 AS id, 'alice' AS name" + "MODEL (description 'Test model stg_orders.', materialized table);\n\nSELECT 1 AS id, 'alice' AS name" ), "models/dim_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' + 'MODEL (description "Test model dim_orders.", materialized table);\n\nSELECT id, name FROM __ref("stg_orders")' ), "tests/unit/test_stg_orders.sql": _FAILING_TEST_SQL, }, diff --git a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py index 0463e6038c..8efa445fc4 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py @@ -37,7 +37,7 @@ def test_given_measurement_outcomes_when_building_then_gates_and_persists_canoni connection: Any, ) -> None: model_sql = """ - MODEL ( + MODEL (description "Test model.", materialized table, audits [ configured_rate ( diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index d853d2d81c..ebac3b9955 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -59,10 +59,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -109,12 +109,12 @@ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -171,10 +171,10 @@ project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -326,12 +326,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -390,12 +390,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_extracts\n" + " - name: raw_customer_region_extracts\n description: Test source raw_customer_region_extracts.\n" " schema: main\n" " table: raw_customer_region_extracts\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -451,12 +451,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -501,12 +501,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_changes\n" + " - name: raw_customer_changes\n description: Test source raw_customer_changes.\n" " schema: main\n" " table: raw_customer_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -569,12 +569,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_changes\n" + " - name: raw_customer_region_changes\n description: Test source raw_customer_region_changes.\n" " schema: main\n" " table: raw_customer_region_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -633,12 +633,12 @@ def test_given_current_state_timestamp_snapshot_when_building_then_tracks_histor "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -753,10 +753,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -782,10 +782,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -807,10 +807,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -834,10 +834,10 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -862,12 +862,12 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_extracts\n" + " - name: raw_customer_extracts\n description: Test source raw_customer_extracts.\n" " schema: main\n" " table: raw_customer_extracts\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -899,12 +899,12 @@ def test_given_historical_timestamp_snapshot_when_building_then_uses_updated_his "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_changes\n" + " - name: raw_customer_changes\n description: Test source raw_customer_changes.\n" " schema: main\n" " table: raw_customer_changes\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -985,10 +985,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1030,12 +1030,12 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1089,10 +1089,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1132,10 +1132,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1175,10 +1175,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1218,12 +1218,12 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1283,10 +1283,10 @@ def test_given_invalid_timestamp_snapshot_source_when_building_then_fails_before project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1415,12 +1415,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1473,12 +1473,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_daily\n" + " - name: raw_customer_region_daily\n description: Test source raw_customer_region_daily.\n" " schema: main\n" " table: raw_customer_region_daily\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1532,12 +1532,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1577,12 +1577,12 @@ def test_given_current_state_check_snapshot_when_building_then_tracks_checked_ch "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_region_daily\n" + " - name: raw_customer_region_daily\n description: Test source raw_customer_region_daily.\n" " schema: main\n" " table: raw_customer_region_daily\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy check,\n" @@ -1688,10 +1688,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1716,12 +1716,12 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_daily\n" + " - name: raw_customer_daily\n description: Test source raw_customer_daily.\n" " schema: main\n" " table: raw_customer_daily\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1749,10 +1749,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1773,10 +1773,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1796,10 +1796,10 @@ def test_given_historical_check_snapshot_when_building_then_tracks_observed_hist project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1870,10 +1870,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -1915,10 +1915,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -1956,10 +1956,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -1993,10 +1993,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2031,10 +2031,10 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n table: raw_customers\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2072,12 +2072,12 @@ def test_given_invalid_check_snapshot_source_when_building_then_fails_before_tar "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customer_regions\n" + " - name: raw_customer_regions\n description: Test source raw_customer_regions.\n" " schema: main\n" " table: raw_customer_regions\n" ), "models/customer_region_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_region_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id, region],\n" " snapshot_strategy timestamp,\n" @@ -2168,11 +2168,11 @@ def test_given_current_state_snapshot_when_source_row_disappears_then_hard_delet initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2184,11 +2184,11 @@ def test_given_current_state_snapshot_when_source_row_disappears_then_hard_delet changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2273,11 +2273,11 @@ def test_given_snapshot_source_adds_column_when_building_then_appends_target_col initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2295,11 +2295,11 @@ def test_given_snapshot_source_adds_column_when_building_then_appends_target_col changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2391,11 +2391,11 @@ def test_given_contract_snapshot_adds_declared_column_when_building_then_require initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2413,11 +2413,11 @@ def test_given_contract_snapshot_adds_declared_column_when_building_then_require changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML_CONFIRM_SCHEMA_CHANGE, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " contract enforced,\n" " columns (\n" @@ -2510,11 +2510,11 @@ def test_given_confirmed_contract_snapshot_adds_declared_column_when_building_th initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2526,11 +2526,11 @@ def test_given_confirmed_contract_snapshot_adds_declared_column_when_building_th changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2606,11 +2606,11 @@ def test_given_wildcard_check_snapshot_source_adds_column_when_building_then_req initial_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2622,11 +2622,11 @@ def test_given_wildcard_check_snapshot_source_adds_column_when_building_then_req changed_project_files={ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( - "sources:\n - name: raw_customers\n schema: main\n" + "sources:\n - name: raw_customers\n description: Test source raw_customers.\n schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2710,12 +2710,12 @@ def test_given_confirmed_wildcard_schema_change_when_building_then_appends_targe "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2804,12 +2804,12 @@ def test_given_hard_delete_snapshot_when_duplicate_keys_fail_then_target_history "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -2872,12 +2872,12 @@ def test_given_check_snapshot_with_initial_valid_from_when_building_then_uses_co "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -2938,12 +2938,12 @@ def test_given_timestamp_snapshot_with_updated_at_initial_validity_when_building "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -3004,12 +3004,12 @@ def test_given_timestamp_snapshot_with_execution_initial_validity_when_building_ "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -3072,12 +3072,12 @@ def test_given_check_snapshot_with_execution_initial_validity_when_building_then "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy timestamp,\n" @@ -3130,12 +3130,12 @@ def test_given_snapshot_initial_validity_config_when_planning_then_plan_entry_pr "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" @@ -3203,12 +3203,12 @@ def test_given_check_snapshot_with_final_audit_when_building_then_runs_audit( "sqlbuild_project.toml": _PROJECT_YML, "sources/raw.yml": ( "sources:\n" - " - name: raw_customers\n" + " - name: raw_customers\n description: Test source raw_customers.\n" " schema: main\n" " table: raw_customers\n" ), "models/customer_snapshot.sql": ( - "MODEL (\n" + "MODEL (description 'Test model customer_snapshot.',\n" " materialized snapshot,\n" " unique_key [customer_id],\n" " snapshot_strategy check,\n" diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index 625fb41eb0..08d4ea960a 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -25,11 +25,11 @@ 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "demo.duckdb"\n' ), "models/customer_totals.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model customer_totals.', materialized table);\n\n" "SELECT customer_id, SUM(amount) AS total FROM raw_orders GROUP BY customer_id" ), "models/orders.sql": ( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized incremental,\n" " incremental_strategy append,\n" " cursor updated_at,\n" diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index b13418d2be..b7d7d40518 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -169,10 +169,11 @@ def build_authored_cte_project_files() -> dict[str, str]: "sql_analysis = true\n" ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/stg_orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model stg_orders.', materialized table);\n\n" "WITH base AS (\n" " -- a comment with an unmatched ) parenthesis\n" " SELECT id AS order_id, amount, 'it''s (fine)' AS note /* ( */\n" @@ -182,7 +183,7 @@ def build_authored_cte_project_files() -> dict[str, str]: "SELECT order_id, amount, note, tag FROM tagged\n" ), "models/orders.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model orders.', materialized table);\n\n" "WITH totals AS (\n" ' WITH ranked AS (SELECT order_id, amount, tag FROM __ref("stg_orders"))\n' " SELECT order_id, amount * 2 AS doubled, tag FROM ranked\n" @@ -253,10 +254,11 @@ def build_terminated_model_project_files() -> dict[str, str]: 'database = "terminated.duckdb"\n\n[settings]\nsql_analysis = true\n' ), "sources/raw.yml": ( - "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" ), "models/m.sql": ( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model m.', materialized table);\n\n" 'WITH a AS (SELECT id FROM __source("raw_orders"))\n' "SELECT id FROM a;\n" ), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 95931ffd37..07151eff97 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -33,7 +33,7 @@ def build_sqlbuild_project_with_manifest( sql: str for model_name, sql in model_sql_by_name.items(): project_dir.joinpath(f"models/{model_name}.sql").write_text( - f"MODEL ();\n\n{sql}\n", encoding="utf-8" + f"MODEL (description 'Test model.');\n\n{sql}\n", encoding="utf-8" ) project_dir.joinpath("target/manifest.json").write_text( manifest_source.read_text(encoding="utf-8"), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index fb053289c5..ed7eef6862 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -26,7 +26,7 @@ [ RealDbtManifestCompileTestCase( description="validates and preserves SQLBuild dbt ref from real dbt manifest", - sqlbuild_model_sql='MODEL ();\n\nselect order_id from __dbt_ref("stg_orders")\n', + sqlbuild_model_sql='MODEL (description "Test model.");\n\nselect order_id from __dbt_ref("stg_orders")\n', expected_compiled_sql='select order_id from __dbt_ref("stg_orders")', ), ], diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py index ba7d7f2900..f161626d30 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py @@ -25,7 +25,7 @@ description="plans dbt-only tag selector and skips SQLBuild work", args=("--select", "tag:nightly"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=(), expected_sqlbuild_command_argvs=(), @@ -42,7 +42,7 @@ description="plans mixed dbt and SQLBuild tag selector matches", args=("--select", "tag:nightly"), sqlbuild_model_contents_by_relative_path={ - "tagged_orders.sql": "MODEL (tags [nightly]);\n\nselect 1 as order_id\n", + "tagged_orders.sql": "MODEL (description 'Test model tagged_orders.', tags [nightly]);\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("tagged_orders",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "tagged_orders"),), @@ -60,10 +60,10 @@ args=("--select", "tag:nightly+"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders", "mart_orders"), @@ -104,7 +104,7 @@ description="plans SQLBuild-only tag selector and skips dbt work", args=("--select", "tag:sqb_only"), sqlbuild_model_contents_by_relative_path={ - "sqb_only.sql": "MODEL (tags [sqb_only]);\n\nselect 1 as order_id\n", + "sqb_only.sql": "MODEL (description 'Test model sqb_only.', tags [sqb_only]);\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("sqb_only",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "sqb_only"),), @@ -121,9 +121,9 @@ description="applies SQLBuild tag excludes after selection", args=("--select", "tag:sqb_only", "--exclude", "tag:deprecated"), sqlbuild_model_contents_by_relative_path={ - "sqb_only.sql": "MODEL (tags [sqb_only]);\n\nselect 1 as order_id\n", + "sqb_only.sql": "MODEL (description 'Test model sqb_only.', tags [sqb_only]);\n\nselect 1 as order_id\n", "deprecated_orders.sql": ( - "MODEL (tags [sqb_only, deprecated]);\n\nselect 2 as order_id\n" + "MODEL (description 'Test model deprecated_orders.', tags [sqb_only, deprecated]);\n\nselect 2 as order_id\n" ), }, expected_sqlbuild_model_names=("sqb_only",), @@ -141,7 +141,7 @@ description="plans SQLBuild-only model and skips dbt command work", args=("--select", "local_only"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("local_only",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "local_only"),), @@ -159,10 +159,10 @@ args=("--select", "+downstream_orders"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders",), @@ -197,10 +197,10 @@ args=("--select", "fact_orders+"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders", "mart_orders"), @@ -223,10 +223,10 @@ args=("--select", "fact_orders+", "--exclude", "mart_orders"), sqlbuild_model_contents_by_relative_path={ "downstream_orders.sql": ( - 'MODEL ();\n\nselect order_id from __dbt_ref("fact_orders")\n' + 'MODEL (description "Test model downstream_orders.");\n\nselect order_id from __dbt_ref("fact_orders")\n' ), "mart_orders.sql": ( - 'MODEL ();\n\nselect order_id from __ref("downstream_orders")\n' + 'MODEL (description "Test model mart_orders.");\n\nselect order_id from __ref("downstream_orders")\n' ), }, expected_sqlbuild_model_names=("downstream_orders",), @@ -246,7 +246,7 @@ description="routes explicit model path selector for real SQLBuild project", args=("--select", "path:models/marts"), sqlbuild_model_contents_by_relative_path={ - "marts/mart_orders.sql": "MODEL ();\n\nselect 1 as order_id\n", + "marts/mart_orders.sql": "MODEL (description 'Test model mart_orders.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("mart_orders",), expected_sqlbuild_command_argvs=(("sqb", "plan", "--select", "mart_orders"),), @@ -264,7 +264,7 @@ description="preserves routed SQLBuild cursor args in real plan argv", args=("--select", "local_only", "--start-cursor-int", "10"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=("local_only",), expected_sqlbuild_command_argvs=( @@ -283,7 +283,7 @@ description="returns stable no-work plan for unmatched selectors", args=("--select", "does_not_exist"), sqlbuild_model_contents_by_relative_path={ - "local_only.sql": "MODEL ();\n\nselect 1 as order_id\n", + "local_only.sql": "MODEL (description 'Test model local_only.');\n\nselect 1 as order_id\n", }, expected_sqlbuild_model_names=(), expected_sqlbuild_command_argvs=(), diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py index 2e774c0b65..7d8cb469e7 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/inspection/helpers.py @@ -752,9 +752,9 @@ class SyntheticSnowflakeProject: incremental_model_names: tuple[str, ...] -_TABLE_MODEL_HEADER: str = "MODEL (\n materialized table,\n);\n\n" +_TABLE_MODEL_HEADER: str = "MODEL (description 'Test model.',\n materialized table,\n);\n\n" _INCREMENTAL_MODEL_HEADER: str = ( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized incremental,\n" " incremental_strategy delete_insert,\n" " cursor ordered_at,\n" @@ -953,6 +953,7 @@ def _sources_yml(*, database: str, source_schema: str, sources: int) -> str: lines.extend( ( f" - name: raw_orders_{index}", + f" description: Test source raw_orders_{index}.", f" database: {database.lower()}", f" schema: {source_schema}", f" table: orders_{index}", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py index fea5b2ad34..5d8df636e4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py @@ -68,7 +68,7 @@ def test_given_referenced_environment_change_when_fingerprinting_then_invalidate description="SQL output context preserves structural cache availability", relative_path="models/orders.sql", config=( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" "SELECT '${CTX:run_id}' AS invocation_id, 1 AS order_id\n" ), expected_available=True, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py index 0726159e65..74dab704e9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py @@ -55,7 +55,7 @@ def build_project_with_scenarios(project_dir: Path) -> CompiledProject: scenario_file=DiscoveredSqlScenarioFile( file_path=project_dir / relative_path, relative_path=Path(relative_path), - contents="SCENARIO();\n\nSELECT 1\n", + contents="SCENARIO(description 'Test scenario.');\n\nSELECT 1\n", header_values={}, sql_body="SELECT 1", name=scenario_name, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 5434cc973c..0ae7db5081 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -93,7 +93,7 @@ def prepare_static_compile_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized view);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) return project_dir @@ -533,18 +533,21 @@ def prepare_rule_gated_compile_project(root: Path, *, test_header: str) -> Path: (project_dir / "sources").mkdir() (project_dir / "sources" / "raw.yml").write_text( "sources:\n" - " - name: raw_orders\n schema: main\n table: raw_orders\n" - " - name: raw_refunds\n schema: main\n table: raw_refunds\n", + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " - name: raw_refunds\n description: Test source raw_refunds.\n" + " schema: main\n table: raw_refunds\n", encoding="utf-8", ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (materialized view);\n\n" + "MODEL (description 'Test model orders.', materialized view);\n\n" 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n', encoding="utf-8", ) (project_dir / "models" / "bad_star.sql").write_text( - 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n', + 'MODEL (description "Test model bad_star.", ' + 'materialized view);\n\nSELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) (project_dir / "tests" / "unit").mkdir(parents=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 252ef119be..dd8663809e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -280,7 +280,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " Wrote: target/compiled/", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " order_id (),\n" @@ -305,7 +305,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " = help: change the declared type or cast the expression explicitly", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " amount_cents (type INTEGER),\n" @@ -327,7 +327,7 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d " = help: add the column to MODEL(columns) or remove it from the SELECT list", ), model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " contract enforced,\n" " columns (\n" @@ -385,7 +385,7 @@ def test_given_contract_errors_when_running_compile_then_reports_diagnostics( expected_line=5, expected_column=5, model_sql=( - "MODEL (\n" + "MODEL (description 'Test model.',\n" " materialized view,\n" " columns (\n" " order_id (),\n" @@ -468,7 +468,7 @@ def test_given_explicit_column_contract_mode_when_compiling_column_audit_then_au encoding="utf-8", ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " materialized view,\n" " columns (\n" " customer_id (audits [not_null]),\n" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py index a8aa26f04d..144dcae048 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py @@ -48,7 +48,7 @@ def prepare_static_dag_project(root: Path) -> Path: encoding="utf-8", ) (models_dir / "orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", encoding="utf-8", ) return project_dir @@ -67,7 +67,8 @@ def prepare_advisory_placement_dag_project(root: Path) -> Path: constants_dir.mkdir() (constants_dir / "value.sql").write_text("CONSTANT (name value, value 1);\n", encoding="utf-8") (project_dir / "models" / "orders.sql").write_text( - 'MODEL (materialized table);\n\nSELECT @const("value") AS order_id\n', + 'MODEL (description "Test model orders.", ' + 'materialized table);\n\nSELECT @const("value") AS order_id\n', encoding="utf-8", ) return project_dir @@ -98,6 +99,7 @@ def prepare_python_dag_project(root: Path) -> Path: " meta={'owner': 'data'},", ")", "def prepare_orders(ctx):", + " '''Test task prepare_orders.'''", " return ctx.result(payload={'rows': 1}, metadata={'source': 'fixture'})", ) ) @@ -115,6 +117,7 @@ def prepare_python_dag_project(root: Path) -> Path: " columns=[{'name': 'order_id', 'type': 'integer'}],", ")", "def warehouse_export(ctx):", + " '''Test loader warehouse_export.'''", " return [{'order_id': 1}]", ) ) @@ -176,6 +179,7 @@ def prepare_python_dag_project(root: Path) -> Path: "", "@check(depends_on=warehouse_export, tags=['loader'])", "def check_loader_export(ctx):", + " '''Test check check_loader_export.'''", " return ctx.pass_(message='loader rows available')", ) ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 7d34289ba8..a062c17353 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -154,6 +154,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit "kind": "python_check", "name": "check_loader_export", "checked_asset_ids": ["loader:warehouse_export"], + "description": "Test check check_loader_export.", "path": "python/checks/check_loader_export.py", "severity": "error", "tags": ["loader"], diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index cae1745351..28b16aacf7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -21,6 +21,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import RequiredDescriptionInputs @dataclass(frozen=True) @@ -1099,3 +1100,32 @@ class CompactLineageCacheEncodingTestCase: output_column: str resource_names: dict[int, str] expected_lineage_columns: int = 2 + + +@dataclass(frozen=True) +class RequiredDescriptionUnitCase: + """One resource kind checked by the required-description compile rule.""" + + description: str + build: Callable[[str | None], RequiredDescriptionInputs] + expected_message: str + expected_location: tuple[str, int, int] + + +@dataclass(frozen=True) +class DescriptionInputClassificationCase: + """Discovered input fields split between required and exempt description kinds.""" + + description: str + required: frozenset[str] + exempt: frozenset[str] + expected_fields: frozenset[str] + + +@dataclass(frozen=True) +class RequiredDescriptionLocationCase: + """Where a P010 finding points when comments or docstrings precede the declaration.""" + + description: str + build: Callable[[], RequiredDescriptionInputs] + expected_location: tuple[str, int, int] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py index 03f6de49dd..a88b55a540 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_model_config_reuse.py @@ -45,7 +45,7 @@ def test_given_static_column_contracts_when_attaching_then_reuses_only_base_conf """.strip() + "\n", "models/customers.sql": """ -MODEL ( +MODEL (description "Test model customers.", columns (customer_id (type INTEGER, nullable false, description "Customer id")), ); @@ -53,7 +53,7 @@ def test_given_static_column_contracts_when_attaching_then_reuses_only_base_conf """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns (order_id (type BIGINT, nullable true, description "Order id")), ); @@ -118,7 +118,7 @@ def test_given_templated_columns_when_attaching_then_resolves_each_model_context """.strip() + "\n", "models/customers.sql": """ -MODEL ( +MODEL (description "Test model customers.", columns ( customer_id ( type ${identifier_type}, @@ -131,7 +131,7 @@ def test_given_templated_columns_when_attaching_then_resolves_each_model_context """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id ( type ${identifier_type}, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 52e442f5dd..18a979f4fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,6 +1,6 @@ import sqlite3 from collections.abc import Callable -from dataclasses import replace +from dataclasses import dataclass, replace from pathlib import Path from types import MappingProxyType from typing import cast @@ -15,6 +15,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, ) +from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( + missing_description_diagnostics, +) from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros, load_project_macros, @@ -34,8 +37,12 @@ CompiledProject, CompiledRelationLocation, CompiledSqlTest, + CompileModelInput, CompileProjectInputs, CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlFunctionInput, DeclarationExpansionContext, DeclarationResolutionContext, DeclarationRuntimeProjection, @@ -43,9 +50,22 @@ LoadedMacro, MacroContext, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import ( + DiscoveredLoaderFunction, + DiscoveredMacroFile, + DiscoveredProjectInputs, + DiscoveredProvider, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, + DiscoveredTaskFunction, +) from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -63,6 +83,14 @@ VisibilityRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind +from sqlbuild.providers import Provider +from sqlbuild.spec.contracts.models import ( + LocalConfig, + ProjectConfig, + SchemaModelEntry, + SchemaSeedEntry, + SourceEntry, +) from sqlbuild.sql_values.types import CollectionRendering DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( @@ -278,7 +306,7 @@ def singular_audit_files(*, base: dict[str, str], sql: str) -> dict[str, str]: def model_header(*, key: str, value: str) -> str: """Return a one-column model whose header sets one key.""" - return f"MODEL ({key} {value});\nSELECT 1 AS order_id" + return f"MODEL (description 'Test model check.', {key} {value});\nSELECT 1 AS order_id" def compile_and_assemble(*, project_dir: Path) -> CompiledProject: @@ -295,7 +323,7 @@ def gate_audit(*, read: str) -> str: return f"AUDIT ();\nSELECT s.* FROM @relation s LEFT JOIN {read} a USING (code)" -def gate_model(*, sql: str, header: str = "MODEL ();") -> str: +def gate_model(*, sql: str, header: str = "MODEL (description 'Test model.');") -> str: """Return a model with the given header.""" return f"{header}\n{sql}" @@ -315,7 +343,7 @@ def execution_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, st def inline_sql_hook_header(sql: str) -> str: """Return a MODEL header running one inline SQL pre-hook.""" - return f"MODEL (pre_hooks [inline_sql('{sql}')]);" + return f"MODEL (description 'Test model.', pre_hooks [inline_sql('{sql}')]);" def lineage_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, str]]: @@ -419,7 +447,7 @@ def python_loader_source(*, depends_on: str, body: str) -> str: "from sqlbuild.loaders import loader\n\n\n" "@loader\n" "def raw_regions(ctx):\n" - " return [{'id': 1}]\n\n\n" + " '''Test loader raw_regions.'''\n return [{'id': 1}]\n\n\n" f"@loader(depends_on=[{depends_on}])\n" "def raw_customers(ctx):\n" f"{body}" @@ -485,3 +513,251 @@ def stored_analysis_contents(*, context: AnalysisCacheContext, cache_key: str) - "SELECT payload FROM model_analysis WHERE cache_key = ?", (cache_key,) ).fetchone() return row[0] + + +REQUIRED_DESCRIPTION_BASE_INPUTS: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), +) + + +@dataclass(frozen=True) +class RequiredDescriptionInputs: + """Compile inputs passed to the required-description check.""" + + discovered_inputs: DiscoveredProjectInputs = REQUIRED_DESCRIPTION_BASE_INPUTS + model_inputs: tuple[CompileModelInput, ...] = () + seed_inputs: tuple[CompileSeedInput, ...] = () + source_inputs: tuple[CompileSourceInput, ...] = () + function_inputs: tuple[CompileSqlFunctionInput, ...] = () + + +class OrdersApi(Provider): + """Client for the orders API.""" + + +class UndescribedApi(Provider): + pass + + +_PROVIDER_CLASS_BY_UNDESCRIBED: dict[bool, type[Provider]] = { + False: OrdersApi, + True: UndescribedApi, +} + + +def refresh_exports_task(ctx: object) -> None: + del ctx + + +def documented_returns_loader(ctx: object) -> None: + """Returned orders from the support desk.""" + + del ctx + + +def model_description_inputs( + description: str | None, + *, + contents: str = "-- totals\nMODEL (materialized table);\nSELECT 1 AS id\n", +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + model_inputs=( + CompileModelInput( + model_file=DiscoveredSqlModelFile( + file_path=Path("/project/models/order_totals.sql"), + relative_path=Path("models/order_totals.sql"), + contents=contents, + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="SELECT 1 AS id", + ), + schema_entry=SchemaModelEntry(name="order_totals", description=description), + ), + ) + ) + + +def scenario_description_inputs( + description: str | None, *, contents: str = "\nSCENARIO ();\nSELECT 1\n" +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + scenario_files=( + DiscoveredSqlScenarioFile( + file_path=Path("/project/tests/scenarios/orders__paid.sql"), + relative_path=Path("tests/scenarios/orders__paid.sql"), + contents=contents, + header_values={"description": description}, + sql_body="SELECT 1", + name="orders__paid", + ), + ), + ) + ) + + +def seed_description_inputs(description: str | None) -> RequiredDescriptionInputs: + contents: str = "seeds:\n - name: product_types\n" + return RequiredDescriptionInputs( + seed_inputs=( + CompileSeedInput( + seed_file=DiscoveredSeedFile( + file_path=Path("/project/seeds/product_types.csv"), + relative_path=Path("seeds/product_types.csv"), + ), + schema_entry=SchemaSeedEntry(name="product_types", description=description), + schema_file=DiscoveredSchemaFile( + file_path=Path("/project/seeds/product_types.yml"), + relative_path=Path("seeds/product_types.yml"), + contents=contents, + model_entries=(), + seed_entries=(), + ), + ), + ) + ) + + +def returns_source_input(*, description: str | None, loader: str | None) -> CompileSourceInput: + return CompileSourceInput( + source_entry=SourceEntry(name="raw_returns", description=description, loader=loader), + source_file=DiscoveredSourceFile( + file_path=Path("/project/sources/raw.yml"), + relative_path=Path("sources/raw.yml"), + contents="sources:\n - name: raw_orders\n description: Orders\n" + " - name: raw_returns\n", + source_entries=(), + ), + ) + + +def source_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + source_inputs=(returns_source_input(description=description, loader=None),) + ) + + +def returns_loader(description: str | None) -> DiscoveredLoaderFunction: + return DiscoveredLoaderFunction( + file_path=Path("/project/python/loaders/raw_returns.py"), + relative_path=Path("python/loaders/raw_returns.py"), + name="raw_returns", + function=documented_returns_loader, + description=description, + ) + + +def loader_backed_source_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + source_inputs=(returns_source_input(description=None, loader="raw_returns"),), + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) + ), + ) + + +def loader_only_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) + ) + ) + + +def function_description_inputs( + description: str | None, + *, + contents: str = "FUNCTION (returns BOOLEAN);\n\nTRUE\n", + relative_path: Path = Path("functions/sql/is_large_order.sql"), + language: FunctionLanguage = FunctionLanguage.SQL, +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + function_inputs=( + CompileSqlFunctionInput( + function_file=DiscoveredSqlFunctionFile( + file_path=Path("/project") / relative_path, + relative_path=relative_path, + contents=contents, + header_values={}, + body_sql="TRUE", + ), + name="is_large_order", + arguments=(), + returns="BOOLEAN", + body_sql="TRUE", + description=description, + language=language, + ), + ) + ) + + +def sql_hook_description_inputs( + description: str | None, *, contents: str = "HOOK ();\nSELECT 1\n" +) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + sql_hook_files=( + DiscoveredSqlHookFile( + file_path=Path("/project/hooks/sql/analyze_orders.sql"), + relative_path=Path("hooks/sql/analyze_orders.sql"), + contents=contents, + header_values={}, + sql_body="SELECT 1", + name="analyze_orders", + description=description, + ), + ), + ) + ) + + +def task_description_inputs(description: str | None) -> RequiredDescriptionInputs: + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + task_functions=( + DiscoveredTaskFunction( + file_path=Path("/project/python/tasks/refresh_exports.py"), + relative_path=Path("python/tasks/refresh_exports.py"), + name="refresh_exports", + function=refresh_exports_task, + description=description, + ), + ), + ) + ) + + +def provider_description_inputs(description: str | None) -> RequiredDescriptionInputs: + provider_class: type[Provider] = _PROVIDER_CLASS_BY_UNDESCRIBED[description is None] + return RequiredDescriptionInputs( + discovered_inputs=replace( + REQUIRED_DESCRIPTION_BASE_INPUTS, + providers=( + DiscoveredProvider( + file_path=Path("/project/providers/orders_api.py"), + relative_path=Path("providers/orders_api.py"), + name="orders_api", + provider_class=provider_class, + settings=provider_class(), + ), + ), + ) + ) + + +def required_description_diagnostics( + inputs: RequiredDescriptionInputs, +) -> tuple[CompilerDiagnostic, ...]: + return missing_description_diagnostics( + discovered_inputs=inputs.discovered_inputs, + model_inputs=inputs.model_inputs, + seed_inputs=inputs.seed_inputs, + source_inputs=inputs.source_inputs, + function_inputs=inputs.function_inputs, + ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index c5d8619aaa..30ab6049fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -53,6 +53,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. contract: enforced expression: "(SELECT 1 AS order_id)" columns: @@ -62,7 +63,7 @@ """.strip() + "\n", "models/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT order_id FROM __source("raw_orders") """.strip() @@ -70,10 +71,10 @@ } _SELECTION_REPO_FILES: dict[str, str] = { "sqlbuild_project.toml": 'name = "selection_demo"\nadapter = "duckdb"\n', - "models/root.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "models/middle.sql": 'MODEL ();\n\nSELECT id FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT id FROM __ref("middle")\n', - "models/unrelated.sql": "MODEL ();\n\nSELECT 2 AS id\n", + "models/root.sql": "MODEL (description 'Test model root.');\n\nSELECT 1 AS id\n", + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT id FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT id FROM __ref("middle")\n', + "models/unrelated.sql": "MODEL (description 'Test model unrelated.');\n\nSELECT 2 AS id\n", } @@ -229,7 +230,7 @@ def test_given_one_changed_model_when_entry_rows_are_absent_then_compact_batch_r with sqlite3.connect(cache_path) as connection: _ = connection.execute("DELETE FROM model_analysis") (tmp_path / "models" / "unrelated.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", + "MODEL (description 'Test model unrelated.');\n\nSELECT 3 AS id\n", encoding="utf-8", ) @@ -290,7 +291,7 @@ def test_given_changed_expanded_sql_when_compiling_then_writes_a_new_analysis_ob ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - changed_sql = 'MODEL ();\n\nSELECT order_id + 1 AS order_id FROM __source("raw_orders")\n' + changed_sql = 'MODEL (description "Test model.");\n\nSELECT order_id + 1 AS order_id FROM __source("raw_orders")\n' (tmp_path / "models" / "orders.sql").write_text(changed_sql, encoding="utf-8") analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) @@ -766,7 +767,7 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( tmp_path, { **_SELECTION_REPO_FILES, - "models/root.sql": "MODEL ();\n\nSELECT CAST(NULL AS INTEGER) AS id\n", + "models/root.sql": "MODEL (description 'Test model root.');\n\nSELECT CAST(NULL AS INTEGER) AS id\n", }, ) _ = compile_project_with_cache(project_dir=tmp_path) @@ -775,8 +776,8 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( unrelated_path: Path = tmp_path / "models" / "unrelated.sql" unrelated_path.write_text( unrelated_path.read_text(encoding="utf-8").replace( - "MODEL ();", - "MODEL (columns (id (nullable false)));", + "MODEL (description 'Test model unrelated.');", + "MODEL (description 'Test model unrelated.', columns (id (nullable false)));", 1, ), encoding="utf-8", @@ -788,8 +789,8 @@ def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( root_path: Path = tmp_path / "models" / "root.sql" root_path.write_text( root_path.read_text(encoding="utf-8").replace( - "MODEL ();", - "MODEL (columns (id (nullable false)));", + "MODEL (description 'Test model root.');", + "MODEL (description 'Test model root.', columns (id (nullable false)));", 1, ), encoding="utf-8", @@ -818,7 +819,7 @@ def test_given_model_sql_change_with_stable_signature_when_compiling_then_downst analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", + "MODEL (description 'Test model root.');\n\nSELECT 3 AS id\n", encoding="utf-8", ) @@ -843,7 +844,7 @@ def test_given_model_output_change_when_compiling_then_downstream_closure_misses analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) @@ -869,7 +870,9 @@ def test_given_changed_output_signature_when_reanalyzing_downstream_then_uses_de _ = compile_project_with_cache(project_dir=tmp_path) root_path: Path = tmp_path / "models" / "root.sql" original_sql: str = root_path.read_text(encoding="utf-8") - root_path.write_text("MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8") + root_path.write_text( + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8" + ) _ = compile_project_with_cache(project_dir=tmp_path) batcher: Mock = Mock(wraps=assembly_project.analyze_queries_with_compact_polyglot_batch) monkeypatch.setattr(assembly_project, "analyze_queries_with_compact_polyglot_batch", batcher) @@ -896,7 +899,9 @@ def test_given_cached_model_signature_is_restored_when_compiling_then_downstream _ = compile_project_with_cache(project_dir=tmp_path) root_path: Path = tmp_path / "models" / "root.sql" original_sql: str = root_path.read_text(encoding="utf-8") - root_path.write_text("MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8") + root_path.write_text( + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8" + ) _ = compile_project_with_cache(project_dir=tmp_path) analyzer: Mock = Mock(wraps=assembly_project.analyze_columns_and_lineage_with_polyglot) monkeypatch.setattr(assembly_project, "analyze_columns_and_lineage_with_polyglot", analyzer) @@ -922,13 +927,13 @@ def test_given_selected_upstream_change_when_compiling_full_project_then_stale_c tmp_path, { **_SELECTION_REPO_FILES, - "models/middle.sql": 'MODEL ();\n\nSELECT * FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT * FROM __ref("middle")\n', + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT * FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT * FROM __ref("middle")\n', }, ) _ = compile_project_with_cache(project_dir=tmp_path) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) _ = compile_project_with_cache( @@ -958,14 +963,14 @@ def test_given_upstream_interface_change_when_compact_batch_exists_then_dependen tmp_path, { **_SELECTION_REPO_FILES, - "models/middle.sql": 'MODEL ();\n\nSELECT * FROM __ref("root")\n', - "models/leaf.sql": 'MODEL ();\n\nSELECT * FROM __ref("middle")\n', + "models/middle.sql": 'MODEL (description "Test model middle.");\n\nSELECT * FROM __ref("root")\n', + "models/leaf.sql": 'MODEL (description "Test model leaf.");\n\nSELECT * FROM __ref("middle")\n', }, ) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) _ = compile_project_with_cache(project_dir=tmp_path) (tmp_path / "models" / "root.sql").write_text( - "MODEL ();\n\nSELECT 1 AS id, 2 AS extra\n", + "MODEL (description 'Test model root.');\n\nSELECT 1 AS id, 2 AS extra\n", encoding="utf-8", ) @@ -1060,7 +1065,7 @@ def test_given_unselected_invalid_reference_when_compiling_then_live_validation_ write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: repo_files: dict[str, str] = _SELECTION_REPO_FILES | { - "models/unrelated.sql": 'MODEL ();\n\nSELECT id FROM __ref("missing")\n' + "models/unrelated.sql": 'MODEL (description "Test model unrelated.");\n\nSELECT id FROM __ref("missing")\n' } write_repo_files(tmp_path, repo_files) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 86538d67c4..d0e4494d48 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -58,7 +58,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (type VARCHAR), ), @@ -71,6 +71,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: public table: orders """.strip() @@ -78,6 +79,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: code type: VARCHAR @@ -132,10 +134,11 @@ repo_files=base_repo_files() | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", "seeds/schema.yml": """ seeds: - name: order_codes + description: Test seed order_codes. columns: - name: order_id type: INTEGER @@ -209,6 +212,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: code type: VARCHAR @@ -262,6 +266,7 @@ "seeds/lookups.yml": """ seeds: - name: country_codes + description: Test seed country_codes. database: "${if(ENV:USE_SEED_DB, 'seed_db', CTX:destination.database)}" schema: "${CTX:destination.schema}_${seed_schema_suffix}" columns: @@ -300,9 +305,9 @@ 'def normalize_status(value):\n return f"LOWER(TRIM({value}))"\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" + "MODEL (description 'Test model orders.');\n\nSELECT @normalize_status(\"' PAID '\") AS status\n" ), - "models/customers.sql": "MODEL ();\n\nSELECT 'active' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 'active' AS status\n", "tests/unit/test_normalize_status.sql": """ TEST (mode macro, name "normalizes_status"); @@ -354,6 +359,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. managed: true write_strategy: table """.strip() @@ -363,6 +369,7 @@ @loader def raw_orders(ctx): + '''Test loader raw_orders.''' return [{"id": 1}] """.strip() + "\n", @@ -387,7 +394,7 @@ def raw_orders(ctx): | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/format_cents.sql": """ -FUNCTION ( +FUNCTION (description "Test function format_cents.", arguments (amount_cents INTEGER), returns VARCHAR, ); @@ -396,9 +403,9 @@ def raw_orders(ctx): """.strip() + "\n", "models/orders.sql": ( - 'MODEL ();\n\nSELECT __udf("format_cents")(1250) AS formatted\n' + 'MODEL (description "Test model orders.");\n\nSELECT __udf("format_cents")(1250) AS formatted\n' ), - "models/customers.sql": "MODEL ();\n\nSELECT 'active' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 'active' AS status\n", "tests/unit/test_format_cents.sql": """ TEST (mode udf, name "formats_cents"); @@ -438,7 +445,7 @@ def raw_orders(ctx): | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( customer_id INTEGER, @@ -450,7 +457,7 @@ def raw_orders(ctx): """.strip() + "\n", "models/customer_order_summary.sql": ( - "MODEL ();\n\nSELECT customer_id, order_id " + "MODEL (description 'Test model customer_order_summary.');\n\nSELECT customer_id, order_id " 'FROM __table_fn("customer_orders")(42)\n' ), "tests/unit/test_customer_orders.sql": """ @@ -593,7 +600,7 @@ def test_given_named_generic_audit_instances_when_assembling_then_each_name_is_p base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [ expression_is_true (name "positive_revenue", expression "revenue > 0"), expression_is_true (name "bounded_revenue", expression "revenue < 1000"), @@ -666,12 +673,14 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. database: declared_seed_db schema: declared_seed_schema columns: - name: code type: VARCHAR - name: currencies + description: Test seed currencies. columns: - name: code type: VARCHAR @@ -802,12 +811,12 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t AssembleSqlHookValidationTestCase( description="validates named and inline payloads with adapter dialect", model_sql=( - 'MODEL (pre_hooks [sql("vendor_hook")], ' + 'MODEL (description "Test model.", pre_hooks [sql("vendor_hook")], ' 'post_hooks [inline_sql("SELECT [other] FROM [orders]; ' 'SELECT [other] FROM [orders]")]);\n\nSELECT 1 AS id\n' ), named_hook_sql=( - "HOOK ();\n\nSELECT [value] FROM [orders]; SELECT [value] FROM [orders]\n" + "HOOK (description 'Test hook.');\n\nSELECT [value] FROM [orders]; SELECT [value] FROM [orders]\n" ), dialect="tsql", expected_hook_statements=( @@ -818,11 +827,11 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t AssembleSqlHookValidationTestCase( description="skips named and inline Polyglot validation for model opt out", model_sql=( - 'MODEL (sql_validation false, pre_hooks [sql("vendor_hook")], ' + 'MODEL (description "Test model.", sql_validation false, pre_hooks [sql("vendor_hook")], ' 'post_hooks [inline_sql("DBCC CHECKIDENT (orders, RESEED, 0)")]);' "\n\nSELECT 1 AS id\n" ), - named_hook_sql="HOOK ();\n\nDBCC CHECKIDENT ('orders', RESEED, 0)\n", + named_hook_sql="HOOK (description 'Test hook.');\n\nDBCC CHECKIDENT ('orders', RESEED, 0)\n", dialect="tsql", expected_hook_statements=( "DBCC CHECKIDENT ('orders', RESEED, 0)", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py index 861e1d6b35..ded2583fcb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py @@ -28,18 +28,14 @@ sql_analysis = false sql_validation = false """ -_SOURCES_WITH_AUDIT: str = ( - "sources:\n - name: raw_orders\n table: orders\n audits:\n - code_check\n" -) +_SOURCES_WITH_AUDIT: str = "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n audits:\n - code_check\n" _SEED_WITH_AUDIT: str = ( - "seeds:\n - name: order_codes\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: order_codes\n description: Test seed order_codes.\n columns:\n - name: code\n type: VARCHAR\n" " audits:\n - code_check\n" ) -_AUDITED_HEADER: str = "MODEL (audits [code_check]);" -_ORDER_CHECK_HEADER: str = "MODEL (audits [order_check]);" -_ALLOWED_SEED: str = ( - "seeds:\n - name: allowed_codes\n columns:\n - name: code\n type: VARCHAR\n" -) +_AUDITED_HEADER: str = "MODEL (description 'Test model.', audits [code_check]);" +_ORDER_CHECK_HEADER: str = "MODEL (description 'Test model.', audits [order_check]);" +_ALLOWED_SEED: str = "seeds:\n - name: allowed_codes\n description: Test seed allowed_codes.\n columns:\n - name: code\n type: VARCHAR\n" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 88a38806c0..4f1801971c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -45,7 +45,7 @@ def test_given_factory_attachment_when_building_inputs_then_cases_append_with_pr tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", + "models/orders.sql": f"MODEL (description 'Test model orders.', {test_case.model_header}); SELECT 1 AS amount", "audits/generic/expression_is_true.sql": _AUDIT_FILE, "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @@ -113,7 +113,7 @@ def test_given_invalid_factory_attachment_when_building_inputs_then_compile_erro tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", + "models/orders.sql": f"MODEL (description 'Test model orders.', {test_case.model_header}); SELECT 1 AS amount", "python/factories/quality.py": f""" from sqlbuild.audits import AuditCase, audit_factory diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 018008170d..922a47a23a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -134,14 +134,14 @@ def test_given_visibility_record_when_rebinding_then_projects_new_consumer_indep "global declaration", {"constants/value.sql": "CONSTANT (name value, value 1);"}, "models/domain/child/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 1 AS value", ), ScopedDeclarationCompileTestCase( "inherited declaration", {"models/domain/constants/value.sql": "CONSTANT (name value, value 2);"}, "models/domain/child/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 2 AS value", ), ScopedDeclarationCompileTestCase( @@ -153,21 +153,23 @@ def test_given_visibility_record_when_rebinding_then_projects_new_consumer_indep ), }, "models/domain/child/grandchild/orders.sql", - ('MODEL ();\nSELECT @const("parent_value") + @const("child_value") AS value\n'), + ( + 'MODEL (description "Test model orders.");\nSELECT @const("parent_value") + @const("child_value") AS value\n' + ), "SELECT 2 + 3 AS value", ), ScopedDeclarationCompileTestCase( "local declaration", {"models/domain/_constants/value.sql": "CONSTANT (name value, value 4);"}, "models/domain/orders.sql", - 'MODEL ();\nSELECT @const("value") AS value\n', + 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value\n', "SELECT 4 AS value", ), ScopedDeclarationCompileTestCase( "private declaration", {}, "models/domain/orders.sql", - 'MODEL (constants (_value 5));\nSELECT @const("_value") AS value\n', + 'MODEL (description "Test model orders.", constants (_value 5));\nSELECT @const("_value") AS value\n', "SELECT 5 AS value", ), ), @@ -198,7 +200,7 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "sibling declaration", { "models/one/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/two/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/two/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ("known but inaccessible", "models/one/_constants/value.sql", "models/two/orders.sql"), ), @@ -206,19 +208,21 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "descendant declaration", { "models/domain/child/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ("known but inaccessible", "models/domain/child/_constants/value.sql"), ), ScopedDeclarationErrorTestCase( "unknown declaration", - {"models/orders.sql": 'MODEL ();\nSELECT @const("missing") AS value'}, + { + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("missing") AS value' + }, ("Unknown constant 'missing'", "Visible constants: none"), ), ScopedDeclarationErrorTestCase( "model private does not export to test", { - "models/orders.sql": "MODEL (constants (_value 1));\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.', constants (_value 1));\nSELECT 1 AS value", "tests/unit/orders.sql": ( 'TEST ();\nWITH __expected__orders AS (SELECT @const("_value") AS value) ' "SELECT 1" @@ -230,7 +234,7 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib "filename resemblance without expected relationship", { "models/domain/_constants/value.sql": "CONSTANT (name model_value, value 9);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/orders.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__assert__valid AS (SELECT @const('model_value') WHERE FALSE) SELECT 1" @@ -241,11 +245,11 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib ScopedDeclarationErrorTestCase( "unrelated production scope is not granted", { - "models/orders/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "models/customers/_constants/value.sql": ( "CONSTANT (name customer_value, value 10);" ), - "models/customers/customers.sql": "MODEL ();\nSELECT 1 AS value", + "models/customers/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('customer_value') AS value) SELECT 1" @@ -278,7 +282,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti { "models/domain/_constants/value.sql": "CONSTANT (name value, value 11);", "models/domain/orders.sql": ( - "MODEL (pre_hooks [inline_sql(\"SELECT @const('value')\")]);\nSELECT 1" + "MODEL (description 'Test model orders.', pre_hooks [inline_sql(\"SELECT @const('value')\")]);\nSELECT 1" ), }, lambda inputs: ( @@ -292,8 +296,8 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "named hook definition path", { "models/domain/_sqlbuild/_constants/value.sql": "CONSTANT (name value, value 12);", - "models/domain/_sqlbuild/_hooks/sql/typed.sql": 'HOOK ();\nSELECT @const("value")', - "models/domain/orders.sql": 'MODEL (post_hooks [sql("typed")]);\nSELECT 1', + "models/domain/_sqlbuild/_hooks/sql/typed.sql": 'HOOK (description "Test hook typed.");\nSELECT @const("value")', + "models/domain/orders.sql": 'MODEL (description "Test model orders.", post_hooks [sql("typed")]);\nSELECT 1', }, lambda inputs: ( cast(list[SqlHookEntry], inputs.model_inputs[0].config.values["post_hooks"])[ @@ -306,8 +310,10 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "function definition path", { "functions/sql/domain/_constants/value.sql": "CONSTANT (name value, value 13);", - "functions/sql/domain/value.sql": ('FUNCTION (returns INTEGER);\n@const("value")'), - "models/orders.sql": "MODEL ();\nSELECT 1", + "functions/sql/domain/value.sql": ( + 'FUNCTION (description "Test function value.", returns INTEGER);\n@const("value")' + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, lambda inputs: inputs.sql_function_inputs[0].body_sql, "13", @@ -320,8 +326,8 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti 'AUDIT ();\nSELECT @const("value") FROM __ref("orders") ' 'JOIN __ref("customers") ON TRUE' ), - "models/domain/orders.sql": "MODEL ();\nSELECT 1", - "models/domain/customers.sql": "MODEL ();\nSELECT 1", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", + "models/domain/customers.sql": "MODEL (description 'Test model customers.');\nSELECT 1", }, lambda inputs: inputs.audit_inputs[0].sql_body, 'SELECT 15 FROM __ref("orders") JOIN __ref("customers") ON TRUE', @@ -335,7 +341,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "models/domain/_sqlbuild/_audits/generic/scoped.sql": ( 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE @const("value") = 18' ), - "models/domain/orders.sql": "MODEL (audits [scoped]);\nSELECT 1", + "models/domain/orders.sql": "MODEL (description 'Test model orders.', audits [scoped]);\nSELECT 1", }, lambda inputs: inputs.audit_inputs[0].sql_body, 'SELECT * FROM __ref("orders") WHERE 18 = 18', @@ -348,7 +354,7 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " '__expected__orders AS (SELECT @const("value") AS value) SELECT 1' ), - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, lambda inputs: inputs.test_inputs[0].sql_body, "WITH __ref__orders AS (SELECT 1 AS value), " @@ -359,10 +365,10 @@ def test_given_inaccessible_or_unknown_declaration_when_compiling_then_diagnosti { "tests/scenarios/domain/_constants/value.sql": ("CONSTANT (name value, value 17);"), "tests/scenarios/domain/orders.sql": ( - "SCENARIO ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "SCENARIO (description 'Test scenario orders.');\nWITH __ref__orders AS (SELECT 1 AS value), " '__expected__orders AS (SELECT @const("value") AS value) SELECT 1' ), - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, lambda inputs: inputs.scenario_inputs[0].sql_body, "WITH __ref__orders AS (SELECT 1 AS value), " @@ -395,7 +401,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ ), "models/domain/_constants/local.sql": ("CONSTANT (name local_value, value 3);"), "models/domain/_enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value, 'OPEN' AS state", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value, 'OPEN' AS state", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('inherited_value') + " @@ -413,9 +419,9 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="multiple expected models deterministic union", files={ "models/a/_constants/a.sql": "CONSTANT (name a_value, value 4);", - "models/a/a.sql": "MODEL ();\nSELECT 4 AS value", + "models/a/a.sql": "MODEL (description 'Test model a.');\nSELECT 4 AS value", "models/b/_constants/b.sql": "CONSTANT (name b_value, value 5);", - "models/b/b.sql": "MODEL ();\nSELECT 5 AS value", + "models/b/b.sql": "MODEL (description 'Test model b.');\nSELECT 5 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__a AS (SELECT 1 AS value), " "__expected__b AS (SELECT @const('b_value') AS value), " @@ -432,7 +438,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="test path visibility remains separate from model grant", files={ "models/domain/_constants/model.sql": "CONSTANT (name model_value, value 6);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/_constants/test.sql": "CONSTANT (name test_value, value 7);", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -453,7 +459,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' ), "models/domain/orders.sql": ( - "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + "MODEL (description 'Test model orders.');\nSELECT @discount('10') AS value, @surcharge('10') AS fee" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -473,7 +479,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ 'def surcharge(expression: str) -> str:\n return f"({expression} + 1)"\n' ), "models/domain/orders.sql": ( - "MODEL ();\nSELECT @discount('10') AS value, @surcharge('10') AS fee" + "MODEL (description 'Test model orders.');\nSELECT @discount('10') AS value, @surcharge('10') AS fee" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -490,7 +496,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="model private declaration is not granted", files={ "models/orders.sql": ( - "MODEL (constants (_private 8));\nSELECT @const('_private') AS value" + "MODEL (description 'Test model orders.', constants (_private 8));\nSELECT @const('_private') AS value" ), "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " @@ -503,7 +509,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ ExpectedModelDeclarationGrantTestCase( description="filename resemblance and no expected model grant nothing", files={ - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/orders.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__assert__valid AS (SELECT 1 WHERE FALSE) SELECT 1" @@ -516,9 +522,9 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ description="scenario expected model grant", files={ "models/domain/_constants/value.sql": "CONSTANT (name scenario_value, value 10);", - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/scenarios/check.sql": ( - "SCENARIO ();\nWITH __ref__orders AS (SELECT 1 AS value), " + "SCENARIO (description 'Test scenario check.');\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS (SELECT @const('scenario_value') AS value) SELECT 1" ), }, @@ -568,7 +574,7 @@ def test_given_expected_models_when_compiling_then_public_declarations_are_grant description="expected model provenance is retained", files={ "models/domain/_constants/value.sql": ("CONSTANT (name model_value, value 6);"), - "models/domain/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/domain/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/check.sql": ( "TEST ();\nWITH __ref__orders AS (SELECT 1 AS value), " "__expected__orders AS " @@ -608,15 +614,15 @@ def test_given_relationship_granted_reference_when_compiling_then_usage_retains_ description="exact local placement is accepted", files={ "models/domain/_constants/value.sql": ("CONSTANT (name value, value 1);"), - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ), ScopePlacementCompileTestCase( description="inherited lowest common ancestor is accepted", files={ "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/a/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', - "models/domain/b/customers.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/a/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', + "models/domain/b/customers.sql": 'MODEL (description "Test model customers.");\nSELECT @const("value") AS value', }, ), ), @@ -660,8 +666,8 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu { "sqlbuild_project.toml": _PROJECT_FILE, "models/domain/area/_constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/area/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', - "models/domain/area/customers.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/area/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', + "models/domain/area/customers.sql": 'MODEL (description "Test model customers.");\nSELECT @const("value") AS value', }, ) @@ -681,7 +687,7 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="used global with one owner is rejected as over broad", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_fragment="required exact-owner-private at 'models'", ), @@ -689,14 +695,16 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="unused global is rejected", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", }, expected_fragment="Unused global declaration 'constant:value'", ), ScopePlacementCompileTestCase( description="unused private is rejected", files={ - "models/orders.sql": ("MODEL (constants (_value 1));\nSELECT 1 AS value"), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', constants (_value 1));\nSELECT 1 AS value" + ), }, expected_fragment="Unused private declaration 'constant:model:orders._value'", ), @@ -704,7 +712,7 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu description="inherited declaration used in one directory must be local", files={ "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_fragment="required exact-owner-private at 'models/domain'", ), @@ -737,7 +745,7 @@ def test_given_invalid_declaration_placement_when_assembling_then_it_reports_an_ description="over broad global is advisory when placement enforcement is disabled", files={ "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, expected_model_names=("orders",), expected_diagnostics=(("S024", "warning"),), @@ -777,7 +785,7 @@ def test_given_disabled_placement_enforcement_when_assembling_then_project_and_w files={"models/domain/enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);"}, model_path="models/domain/child/orders.sql", model_sql=( - "MODEL (contract enforced, columns (state (type state)));\n" + "MODEL (description 'Test model orders.', contract enforced, columns (state (type state)));\n" 'SELECT @enum("state").OPEN AS state' ), expected_sql="SELECT 'OPEN' AS state", @@ -814,7 +822,7 @@ def test_given_inherited_enum_when_compiling_contract_then_resolves_from_model_p description="source authored path", files={ "sources/domain/_constants/value.sql": "CONSTANT (name value, value 14);", - "models/orders.sql": "MODEL ();\nSELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, model_path="sources/domain/source.yml", model_sql='SELECT @const("value")', @@ -874,7 +882,7 @@ def test_given_sources_in_same_file_when_reusing_visibility_then_usage_consumers { "sqlbuild_project.toml": _PROJECT_FILE, "sources/domain/_constants/value.sql": "CONSTANT (name value, value 14);", - "models/orders.sql": "MODEL ();\nSELECT 1", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1", }, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) @@ -932,7 +940,7 @@ def test_given_tests_in_same_directory_when_reusing_visibility_then_usage_consum tmp_path, { "sqlbuild_project.toml": _PROJECT_FILE, - "models/orders.sql": "MODEL ();\nSELECT 1 AS value", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value", "tests/unit/domain/_constants/value.sql": "CONSTANT (name value, value 16);", "tests/unit/domain/customer_orders.sql": test_sql, "tests/unit/domain/inventory_orders.sql": test_sql, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index f85ca4e113..33c0796275 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -81,7 +81,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop CONSTANT (name source_name, value "O'Brien"); """, "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", contract enforced, pre_hooks [inline_sql('SELECT @const("min_items") + @const("_offset")')], enums ( @@ -107,7 +107,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop @const("source_name") AS source_name """, "functions/sql/add_threshold.sql": """ -FUNCTION ( +FUNCTION (description "Test function add_threshold.", arguments (value INTEGER), returns INTEGER, ); @@ -117,6 +117,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop "sources/inline.yml": """ sources: - name: inline_values + description: Test source inline_values. expression: | SELECT @const("min_items") AS value """, @@ -132,7 +133,7 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop SELECT 1 """, "tests/scenarios/orders.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario orders."); WITH __ref__orders AS ( SELECT @const("min_items") AS threshold @@ -321,7 +322,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ CONSTANT (name labels, value (GB "Great Britain", FR "France")); """, "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [inline_sql("SELECT 'GB' IN @const('countries')")], post_hooks [sql("typed_hook")], audits [typed_audit], @@ -338,7 +339,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ @const("_local_values") AS local_values """, "functions/sql/is_supported.sql": """ -FUNCTION (arguments (value VARCHAR), returns BOOLEAN); +FUNCTION (description "Test function is_supported.", arguments (value VARCHAR), returns BOOLEAN); value IN @const("countries") """, "hooks/sql/typed_hook.sql": """ @@ -348,6 +349,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ "sources/inline.yml": """ sources: - name: supported_entries + description: Test source supported_entries. expression: | SELECT * FROM entries WHERE country IN @const("countries") """, @@ -358,7 +360,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ SELECT 1 """, "tests/scenarios/orders.sql": """ -SCENARIO (); +SCENARIO (description "Test scenario orders."); WITH __ref__orders AS (SELECT @const("country_array") AS countries), __expected__orders AS (SELECT @const("country_array") AS countries) SELECT 1 @@ -405,7 +407,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/threshold.sql", initial_declaration="CONSTANT (name threshold, value 7);\n", changed_declaration="CONSTANT (name threshold, value 8);\n", - model_sql='MODEL ();\nSELECT @const("threshold") AS threshold\n', + model_sql='MODEL (description "Test model threshold.");\nSELECT @const("threshold") AS threshold\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -414,7 +416,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["FR", "GB"]);\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -423,7 +425,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["GB", "FR", "FR"]);\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -432,7 +434,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value {"GB", "FR"});\n', changed_declaration='CONSTANT (name countries, value {"FR", "GB"});\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=False, expected_metadata_changed=False, ), @@ -445,7 +447,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ changed_declaration=( 'CONSTANT (name labels, value (FR "France", GB "Great Britain"));\n' ), - model_sql='MODEL ();\nSELECT @const("labels") AS labels\n', + model_sql='MODEL (description "Test model.");\nSELECT @const("labels") AS labels\n', expected_query_hash_changed=False, expected_metadata_changed=False, ), @@ -454,7 +456,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value {"GB", "FR"});\n', changed_declaration='CONSTANT (name countries, value {"GB", "HK"});\n', - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model countries.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -463,7 +465,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/labels.sql", initial_declaration='CONSTANT (name labels, value (GB "Great Britain"));\n', changed_declaration='CONSTANT (name labels, value (GB "Britain"));\n', - model_sql='MODEL ();\nSELECT @const("labels") AS labels\n', + model_sql='MODEL (description "Test model labels.");\nSELECT @const("labels") AS labels\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -474,7 +476,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ changed_declaration=( 'CONSTANT (name countries, value ["GB", "FR"], render_as array);\n' ), - model_sql='MODEL ();\nSELECT @const("countries") AS countries\n', + model_sql='MODEL (description "Test model.");\nSELECT @const("countries") AS countries\n', expected_query_hash_changed=True, expected_metadata_changed=False, ), @@ -483,11 +485,13 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ declaration_path="models/_constants/countries.sql", initial_declaration='CONSTANT (name countries, value ["GB", "FR"]);\n', changed_declaration='CONSTANT (name countries, value ["GB", "HK"]);\n', - model_sql="MODEL ();\nSELECT 1 AS value\n", + model_sql="MODEL (description 'Test model countries.');\nSELECT 1 AS value\n", expected_query_hash_changed=False, expected_metadata_changed=False, additional_files={ - "models/z_consumer.sql": ('MODEL ();\nSELECT @const("countries") AS countries\n') + "models/z_consumer.sql": ( + 'MODEL (description "Test model z_consumer.");\nSELECT @const("countries") AS countries\n' + ) }, ), DeclarationFingerprintTestCase( @@ -496,7 +500,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ initial_declaration="ENUM (name state, members [OPEN]);\n", changed_declaration="ENUM (name state, members [OPEN, CLOSED]);\n", model_sql=""" -MODEL ( +MODEL (description "Test model state.", contract enforced, columns (state (type state)), ); @@ -547,7 +551,9 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="unknown enum member", repo_files={ "enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);", - "models/orders.sql": ('MODEL ();\nSELECT @enum("state").MISSING AS state\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\nSELECT @enum("state").MISSING AS state\n' + ), }, expected_error_fragment="Unknown member 'MISSING' for enum 'state'", ), @@ -555,29 +561,31 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="lowercase enum member access", repo_files={ "enums/state.sql": 'ENUM (name state, members (DELIVERY "active"));', - "models/orders.sql": ('MODEL ();\nSELECT @enum("state").active AS state\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\nSELECT @enum("state").active AS state\n' + ), }, expected_error_fragment="Unknown member 'active' for enum 'state'", ), CompileDeclarationsErrorTestCase( description="unknown constant", repo_files={ - "models/orders.sql": 'MODEL ();\nSELECT @const("missing") AS value\n', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("missing") AS value\n', }, expected_error_fragment="Unknown constant 'missing'", ), CompileDeclarationsErrorTestCase( description="foreign private constant", repo_files={ - "models/a.sql": "MODEL (constants (_limit 7));\nSELECT 1 AS value\n", - "models/b.sql": 'MODEL ();\nSELECT @const("_limit") AS value\n', + "models/a.sql": "MODEL (description 'Test model a.', constants (_limit 7));\nSELECT 1 AS value\n", + "models/b.sql": 'MODEL (description "Test model b.");\nSELECT @const("_limit") AS value\n', }, expected_error_fragment="Constant '_limit' is known but inaccessible", ), CompileDeclarationsErrorTestCase( description="non-private model constant name", repo_files={ - "models/orders.sql": "MODEL (constants (limit 7));\nSELECT 1 AS value\n", + "models/orders.sql": "MODEL (description 'Test model orders.', constants (limit 7));\nSELECT 1 AS value\n", }, expected_error_fragment=( "Invalid model-local constant identity 'limit'.*use snake_case '_limit'" @@ -588,7 +596,7 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity repo_files={ "enums/one.sql": "ENUM (name state, members [OPEN]);", "enums/nested/two.sql": "ENUM (name state, members [CLOSED]);", - "models/orders.sql": "MODEL ();\nSELECT 1 AS value\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS value\n", }, expected_error_fragment="Duplicate declaration 'enum:state'", ), @@ -596,7 +604,7 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity description="nested collection cannot render as a portable value list", repo_files={ "constants/groups.sql": "CONSTANT (name groups, value [[1], [2]]);", - "models/orders.sql": 'MODEL ();\nSELECT @const("groups") AS groups\n', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("groups") AS groups\n', }, expected_error_fragment=( "constants/groups.sql constant 'groups'.*adapter 'duckdb'.*value_list" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py new file mode 100644 index 0000000000..79ecef24b8 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -0,0 +1,229 @@ +"""Unit tests for the required-description compile check (P010).""" + +from __future__ import annotations + +import inspect +from dataclasses import asdict, fields +from functools import partial +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.constants import ( + DESCRIPTION_EXEMPT_INPUTS, + DESCRIPTION_REQUIRED_INPUT_KINDS, +) +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import FunctionLanguage +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DescriptionInputClassificationCase, + RequiredDescriptionLocationCase, + RequiredDescriptionUnitCase, +) +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + UndescribedApi, + documented_returns_loader, + function_description_inputs, + loader_backed_source_description_inputs, + loader_only_description_inputs, + model_description_inputs, + provider_description_inputs, + refresh_exports_task, + required_description_diagnostics, + scenario_description_inputs, + seed_description_inputs, + source_description_inputs, + sql_hook_description_inputs, + task_description_inputs, +) + +_UDF_AFTER_MENTIONS: str = ( + '"""Mentions @udf before the decorator."""\n' + "# @udf in a comment\n" + "from sqlbuild.functions import udf\n\n\n" + '@udf(returns="BOOLEAN")\n' + "def main(amount: int) -> bool:\n" + " return amount > 100\n" +) + + +CASES: tuple[RequiredDescriptionUnitCase, ...] = ( + RequiredDescriptionUnitCase( + "model header after a leading comment", + model_description_inputs, + "model 'order_totals' has no description", + ("models/order_totals.sql", 2, 1), + ), + RequiredDescriptionUnitCase( + "scenario header", + scenario_description_inputs, + "scenario 'orders__paid' has no description", + ("tests/scenarios/orders__paid.sql", 2, 1), + ), + RequiredDescriptionUnitCase( + "seed YAML entry", + seed_description_inputs, + "seed 'product_types' has no description", + ("seeds/product_types.yml", 2, 1), + ), + RequiredDescriptionUnitCase( + "source YAML entry among others", + source_description_inputs, + "source 'raw_returns' has no description", + ("sources/raw.yml", 4, 1), + ), + RequiredDescriptionUnitCase( + "source described through its loader", + loader_backed_source_description_inputs, + "source 'raw_returns' has no description", + ("sources/raw.yml", 4, 1), + ), + RequiredDescriptionUnitCase( + "loader without a source declaration", + loader_only_description_inputs, + "loader 'raw_returns' has no description", + ("python/loaders/raw_returns.py", documented_returns_loader.__code__.co_firstlineno, 1), + ), + RequiredDescriptionUnitCase( + "SQL function header", + function_description_inputs, + "function 'is_large_order' has no description", + ("functions/sql/is_large_order.sql", 1, 1), + ), + RequiredDescriptionUnitCase( + "named SQL hook", + sql_hook_description_inputs, + "hook 'analyze_orders' has no description", + ("hooks/sql/analyze_orders.sql", 1, 1), + ), + RequiredDescriptionUnitCase( + "task", + task_description_inputs, + "task 'refresh_exports' has no description", + ("python/tasks/refresh_exports.py", refresh_exports_task.__code__.co_firstlineno, 1), + ), + RequiredDescriptionUnitCase( + "provider", + provider_description_inputs, + "provider 'orders_api' has no description", + ("providers/orders_api.py", inspect.getsourcelines(UndescribedApi)[1], 1), + ), +) + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], + ids=lambda case: case.description, +) +def test_given_undescribed_resource_when_checking_then_reports_one_p010_at_declaration( + test_case: RequiredDescriptionUnitCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build(None) + ) + + assert [ + (item.code, item.message, (str(item.path), item.line, item.column)) for item in diagnostics + ] == [("P010", test_case.expected_message, test_case.expected_location)] + + +@pytest.mark.parametrize( + "test_case", + [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], + ids=lambda case: case.description, +) +def test_given_described_resource_when_checking_then_reports_nothing( + test_case: RequiredDescriptionUnitCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build("Described resource.") + ) + + assert test_case.expected_message not in [item.message for item in diagnostics] + assert diagnostics == () + + +@pytest.mark.parametrize( + "test_case", + [ + DescriptionInputClassificationCase( + description="every discovered input field is required or exempt", + required=frozenset(DESCRIPTION_REQUIRED_INPUT_KINDS), + exempt=frozenset(DESCRIPTION_EXEMPT_INPUTS), + expected_fields=frozenset(item.name for item in fields(DiscoveredProjectInputs)), + ), + ], + ids=lambda case: case.description, +) +def test_given_discovered_input_fields_when_classifying_then_every_field_is_checked_or_exempt( + test_case: DescriptionInputClassificationCase, +) -> None: + assert not test_case.required & test_case.exempt + assert test_case.required | test_case.exempt == test_case.expected_fields + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionLocationCase( + description="MODEL header after a block comment that mentions MODEL (", + build=partial( + model_description_inputs, + None, + contents="/*\nMODEL (old)\n*/\n-- note\nMODEL (materialized table);\nSELECT 1\n", + ), + expected_location=("models/order_totals.sql", 5, 1), + ), + RequiredDescriptionLocationCase( + description="FUNCTION header after a leading comment", + build=partial( + function_description_inputs, + None, + contents="-- returns a flag\nFUNCTION (returns BOOLEAN);\n\nTRUE\n", + ), + expected_location=("functions/sql/is_large_order.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="HOOK header after a leading block comment", + build=partial( + sql_hook_description_inputs, None, contents="/* analyze */\nHOOK ();\nSELECT 1\n" + ), + expected_location=("hooks/sql/analyze_orders.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="SCENARIO header after a leading comment", + build=partial( + scenario_description_inputs, None, contents="-- paid\nSCENARIO ();\nSELECT 1\n" + ), + expected_location=("tests/scenarios/orders__paid.sql", 2, 1), + ), + RequiredDescriptionLocationCase( + description="@udf decorator after docstring and comment mentions", + build=partial( + function_description_inputs, + None, + contents=_UDF_AFTER_MENTIONS, + relative_path=Path("functions/python/is_large_order.py"), + language=FunctionLanguage.PYTHON, + ), + expected_location=("functions/python/is_large_order.py", 6, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_comments_before_declaration_when_checking_then_points_at_declaration( + test_case: RequiredDescriptionLocationCase, +) -> None: + diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( + test_case.build() + ) + + assert [(str(item.path), item.line, item.column) for item in diagnostics] == [ + test_case.expected_location + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py index 5e1598f99d..9e8b4fccbe 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py @@ -31,15 +31,19 @@ sql_validation = false """ _HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py" -_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);' -_COUNTRY_SEED: str = ( - "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n" +_HOOKED_HEADER: str = ( + 'MODEL (description "Test model lookups.", post_hooks [python("refresh_lookup")]);' +) +_COUNTRY_SEED: str = "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n - name: code\n type: VARCHAR\n" +_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n" ) -_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n" _SQL_HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/sql/record_rows.sql" _MACRO_PATH: str = "models/marts/_sqlbuild/_macros/counts.py" _IGNORING_MACRO: str = "def count_rows(relation):\n return 'SELECT 1'\n" -_NAMED_SQL_HOOK_HEADER: str = 'MODEL (post_hooks [sql("record_rows")]);' +_NAMED_SQL_HOOK_HEADER: str = ( + 'MODEL (description "Test model counts.", post_hooks [sql("record_rows")]);' +) @pytest.mark.parametrize( @@ -86,7 +90,7 @@ description="named SQL hook passing a model to a macro that ignores it", files={ _MACRO_PATH: _IGNORING_MACRO, - _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("customers"))\n', + _SQL_HOOK_PATH: 'HOOK (description "Test hook.");\n@count_rows(__ref("customers"))\n', "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"), "models/marts/orders.sql": gate_model( sql="SELECT 1 AS order_id", header=_NAMED_SQL_HOOK_HEADER @@ -199,11 +203,11 @@ def test_given_sql_hook_read_when_compiling_then_lineage_and_selection_ignore_it description="inline and named SQL hooks reading a model built from their model", files={ _MACRO_PATH: _IGNORING_MACRO, - _SQL_HOOK_PATH: 'HOOK ();\n@count_rows(__ref("order_count"))\n', + _SQL_HOOK_PATH: 'HOOK (description "Test hook.");\n@count_rows(__ref("order_count"))\n', "models/marts/orders.sql": gate_model( sql="SELECT 1 AS order_id", header=( - "MODEL (pre_hooks [inline_sql('SELECT * FROM __ref(\"order_rollup\")')], " + "MODEL (description 'Test model orders.', pre_hooks [inline_sql('SELECT * FROM __ref(\"order_rollup\")')], " 'post_hooks [sql("record_rows")]);' ), ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 40e6fc1a11..71a1e83e07 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -60,7 +60,7 @@ def test_given_identical_contract_headers_when_attaching_then_each_model_keeps_i test_case: ExpectedCountTestCase, ) -> None: model_sql: str = """ -MODEL ( +MODEL (description "Test model.", columns (order_id (type INTEGER, audits [not_null])), ); SELECT 1::INTEGER AS order_id @@ -405,7 +405,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac { "sqlbuild_project.toml": _PROJECT_FILE, "models/stg_order_amounts.sql": """ -MODEL ( +MODEL (description "Test model stg_order_amounts.", contract enforced, columns ( customer_id (type INTEGER), @@ -419,7 +419,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac CAST(10.25 AS DECIMAL(12,2)) AS amount """, "models/customer_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -437,7 +437,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac GROUP BY customer_id """, "models/customer_category_amounts_copy.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts_copy.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -522,7 +522,7 @@ def test_given_partial_upstream_schema_when_compiling_dynamic_pivot_then_closure { "sqlbuild_project.toml": _PROJECT_FILE, "models/order_amounts.sql": """ -MODEL ( +MODEL (description "Test model order_amounts.", contract none, columns ( customer_id (type INTEGER), @@ -537,7 +537,7 @@ def test_given_partial_upstream_schema_when_compiling_dynamic_pivot_then_closure CAST(99 AS INTEGER) AS unrelated """, "models/customer_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model customer_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -594,7 +594,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu { "sqlbuild_project.toml": _PROJECT_FILE, "models/order_amounts.sql": """ -MODEL ( +MODEL (description "Test model order_amounts.", contract enforced, columns ( customer_id (type INTEGER), @@ -610,7 +610,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu CAST(99 AS INTEGER) AS unrelated """, "models/open_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model open_category_amounts.", contract none, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -628,7 +628,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu GROUP BY customer_id """, "models/closed_category_amounts.sql": """ -MODEL ( +MODEL (description "Test model closed_category_amounts.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( @@ -826,7 +826,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ "schemas/order.sql": """ SCHEMA (name order, extends missing, columns (id (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="extends unknown schema 'missing'", ), @@ -838,7 +838,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ SCHEMA (name second, extends third, columns (second_id (type INTEGER))); SCHEMA (name third, extends first, columns (third_id (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="first -> second -> third -> first", ), @@ -849,7 +849,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ SCHEMA (name base, columns (order_id (type INTEGER))); SCHEMA (name child, extends base, columns (ORDER_ID (type INTEGER))); """, - "models/orders.sql": "MODEL (); SELECT 1 AS id", + "models/orders.sql": "MODEL (description 'Test model orders.'); SELECT 1 AS id", }, expected_error_fragment="redeclares inherited column 'ORDER_ID'", ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 2fa99d70f4..2afc430668 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -39,24 +39,24 @@ description="expands ordered typed model and macro test cases before macros", repo_files=base_repo_files() | { - "models/orders.sql": "MODEL ();\n\nSELECT 'open' AS status\n", - "models/customers.sql": "MODEL ();\n\nSELECT 1 AS customer_id\n", - "sources/raw.yml": "sources:\n - name: raw_orders\n expression: SELECT 1 AS order_id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 'open' AS status\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nSELECT 1 AS customer_id\n", + "sources/raw.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: SELECT 1 AS order_id\n", "seeds/schema.yml": ( - "seeds:\n - name: country_codes\n columns:\n" + "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n" " - name: code\n type: VARCHAR\n" ), "seeds/country_codes.csv": "code\nGB\n", "models/_constants/offset.sql": "CONSTANT (name offset, value 2);\n", "models/_enums/state.sql": "ENUM (name state, members [OPEN, CLOSED]);\n", "functions/sql/add_one.sql": """ -FUNCTION (arguments (value INTEGER), returns INTEGER); +FUNCTION (description "Test function add_one.", arguments (value INTEGER), returns INTEGER); value + 1 """.strip() + "\n", "functions/sql/by_customer.sql": """ -FUNCTION ( +FUNCTION (description "Test function by_customer.", arguments (customer_id INTEGER), returns table (customer_id INTEGER) ); diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py index 0c4f939cf2..8e3379e8c7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py @@ -79,7 +79,7 @@ def test_given_windows_style_model_path_when_finding_path_default_then_returns_n model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / test_case.model_relative_path, relative_path=Path(test_case.model_relative_path), - contents="MODEL ();\n\nselect 1\n", + contents="MODEL (description 'Test model.');\n\nselect 1\n", header_values={}, header_column_locations={}, output_column_locations={}, @@ -118,7 +118,7 @@ def test_given_ambiguous_globs_when_finding_path_default_then_raises_structured_ model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / test_case.model_relative_path, relative_path=Path(test_case.model_relative_path), - contents="MODEL ();\n\nselect 1\n", + contents="MODEL (description 'Test model.');\n\nselect 1\n", header_values={}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index 13e530baf3..85a6f1f44c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -33,14 +33,16 @@ _DISABLED_PROJECT_FILE: str = _PROJECT_FILE + "\n[references]\nenforce_explicit = false\n" _HOOK_PATH: str = "models/marts/_sqlbuild/_hooks/python/lookups.py" _TASK_PATH: str = "python/tasks/export.py" -_HOOKED_HEADER: str = 'MODEL (post_hooks [python("refresh_lookup")]);' +_HOOKED_HEADER: str = ( + 'MODEL (description "Test model export.", post_hooks [python("refresh_lookup")]);' +) _MODELS: dict[str, str] = { "models/marts/customers.sql": gate_model(sql="SELECT 1 AS customer_id"), "models/marts/orders.sql": gate_model(sql="SELECT 1 AS order_id", header=_HOOKED_HEADER), } _COUNTRY_SEED: dict[str, str] = { "seeds/country_codes.yml": ( - "seeds:\n - name: country_codes\n columns:\n - name: code\n type: VARCHAR\n" + "seeds:\n - name: country_codes\n description: Test seed country_codes.\n columns:\n - name: code\n type: VARCHAR\n" ), "seeds/country_codes.csv": "code\nGB\n", } @@ -51,9 +53,9 @@ _RAW_SOURCES: dict[str, str] = { "sources/raw.yml": ( "sources:\n" - " - name: raw_regions\n managed: true\n write_strategy: table\n" + " - name: raw_regions\n description: Test source raw_regions.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" - " - name: raw_customers\n managed: true\n write_strategy: table\n" + " - name: raw_customers\n description: Test source raw_customers.\n managed: true\n write_strategy: table\n" " columns:\n - name: id\n type: INTEGER\n" ), } diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py index 587c5b3e45..12f6f8acf7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py @@ -30,17 +30,17 @@ sql_validation = false """ _ADVISORY_PROJECT_FILE: str = _PROJECT_FILE + "\n[scopes]\nenforce_placement = false\n" -_ORDERS: str = "MODEL ();\nSELECT 1 AS order_id" -_CUSTOMERS: str = "MODEL ();\nSELECT 1 AS order_id" -_SOURCES: str = "sources:\n - name: raw_orders\n table: orders\n" -_SEED_YAML: str = ( - "seeds:\n - name: order_codes\n columns:\n - name: order_id\n type: INTEGER\n" +_ORDERS: str = "MODEL (description 'Test model.');\nSELECT 1 AS order_id" +_CUSTOMERS: str = "MODEL (description 'Test model.');\nSELECT 1 AS order_id" +_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n" ) -_TABLE_FUNCTION: str = "FUNCTION (returns table (order_id INTEGER));\nSELECT 1 AS order_id" +_SEED_YAML: str = "seeds:\n - name: order_codes\n description: Test seed order_codes.\n columns:\n - name: order_id\n type: INTEGER\n" +_TABLE_FUNCTION: str = "FUNCTION (description 'Test function.', returns table (order_id INTEGER));\nSELECT 1 AS order_id" _GENERIC_AUDIT: str = 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE order_id IS NULL' _SCHEMA: str = "SCHEMA (name order_shape, columns (order_id (type INTEGER)));" -_SQL_HOOK: str = "HOOK ();\nSELECT 1" -_PYTHON_HOOK: str = "from sqlbuild.hooks import hook\n\n\n@hook\ndef mark(ctx):\n return None\n" +_SQL_HOOK: str = "HOOK (description 'Test hook.');\nSELECT 1" +_PYTHON_HOOK: str = "from sqlbuild.hooks import hook\n\n\n@hook\ndef mark(ctx):\n '''Test hook mark.'''\n return None\n" _SINGULAR_BASE_FILES: dict[str, str] = { "models/orders.sql": _ORDERS, "models/customers.sql": _CUSTOMERS, @@ -252,7 +252,7 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc "CONSTANT (name order_limit, value 10);" ), "models/staging/orders_stage.sql": ( - 'MODEL ();\nSELECT @const("order_limit") AS order_id' + 'MODEL (description "Test model orders_stage.");\nSELECT @const("order_limit") AS order_id' ), "models/marts/_sqlbuild/_audits/generic/order_check.sql": ( 'AUDIT ();\nSELECT * FROM __ref("@model") ' @@ -508,7 +508,7 @@ def test_given_invalid_named_declaration_layout_when_compiling_then_error_names_ "ENUM (name order_status, members [OPEN, CLOSED]);" ), "models/staging/orders_stage.sql": ( - 'MODEL ();\nSELECT @enum("order_status").OPEN AS status' + 'MODEL (description "Test model orders_stage.");\nSELECT @enum("order_status").OPEN AS status' ), "models/marts/_sqlbuild/_schemas/order_shape.sql": ( "SCHEMA (name order_shape, columns (status (type order_status)));" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index c1e5616c6a..d4134b031f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -37,8 +37,8 @@ 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_analysis = false\n' "sql_validation = false\n" ), - "models/orders_eu.sql": "MODEL ();\nSELECT 1 AS order_id", - "models/orders_us.sql": "MODEL ();\nSELECT 2 AS order_id", + "models/orders_eu.sql": "MODEL (description 'Test model orders_eu.');\nSELECT 1 AS order_id", + "models/orders_us.sql": "MODEL (description 'Test model orders_us.');\nSELECT 2 AS order_id", } @@ -392,7 +392,7 @@ def test_given_typed_macro_arguments_when_compiling_then_every_argument_is_a_dep _DEPENDENCY_PROJECT_FILES | { "macros/common.py": test_case.macro_file_contents, - "models/order_summary.sql": f"MODEL ();\n{test_case.sql}", + "models/order_summary.sql": f"MODEL (description 'Test model order_summary.');\n{test_case.sql}", }, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 20bbab00bf..f203b5f056 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -31,7 +31,7 @@ "[path_defaults.events]\ncursor_start = 20\n" ), "models/events/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy delete_insert\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy delete_insert\n" "cursor id\ncursor_type integer\ncursor_start 30\n);\n\nSELECT 1 AS id" ), }, @@ -74,7 +74,7 @@ def test_given_cursor_start_layers_when_building_compile_inputs_then_model_uses_ 'cursor_future_max_distance = "disabled"\n' ), "models/events/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy delete_insert\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy delete_insert\n" "cursor event_at\ncursor_type timestamp\ncursor_grain day\n" "cursor_start_max_ahead '0d'\ncursor_start_max_action cap\n" "cursor_future_max_distance '0d'\ncursor_future_action cap\n" @@ -117,7 +117,7 @@ def test_given_cursor_safety_layers_when_compiling_then_model_fields_use_model_p repo_files={ "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy merge\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy merge\n" "cursor_type integer\ncursor_start 100\n);\n\nSELECT 1 AS id" ), }, @@ -128,7 +128,7 @@ def test_given_cursor_safety_layers_when_compiling_then_model_fields_use_model_p repo_files={ "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "models/orders.sql": ( - "MODEL (\nmaterialized incremental\nincremental_strategy merge\ncursor id\n" + "MODEL (description 'Test model orders.',\nmaterialized incremental\nincremental_strategy merge\ncursor id\n" "cursor_type integer\nunique_key [id]\n" "cursor_start '3.14'\n);\n\nSELECT 1 AS id" ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index f320d89c0a..00bdbdcd28 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -128,7 +128,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (type VARCHAR), ), @@ -138,7 +138,7 @@ """.strip() + "\n", "models/staging/nested/orders_enriched.sql": """ -MODEL ( +MODEL (description "Test model orders_enriched.", materialized incremental, incremental_strategy delete_insert, cursor event_time, @@ -156,6 +156,7 @@ "seeds/schema.yml": """ seeds: - name: country_codes + description: Test seed country_codes. columns: - name: country_code type: VARCHAR @@ -167,6 +168,7 @@ "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. schema: public table: orders """.strip() @@ -175,7 +177,7 @@ selected_target=None, cli_vars={"shared": "cli", "cli_only": "present"}, run_id=None, - expected_model_schema_names=(None, "orders"), + expected_model_schema_names=("orders_enriched", "orders"), expected_model_config_values=( { "materialized": "incremental", @@ -220,11 +222,14 @@ ), BuildCompileInputsTestCase( description="allows models with no matching schema metadata", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target=None, cli_vars={}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -241,7 +246,7 @@ repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, contract enforced, columns ( @@ -295,7 +300,7 @@ repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -388,12 +393,12 @@ shared = "local" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target="prod", cli_vars={}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -444,12 +449,12 @@ target = "dev" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -481,12 +486,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -523,12 +528,12 @@ active = "dev" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -565,12 +570,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -604,12 +609,12 @@ warehouse = "dev_wh" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -645,12 +650,12 @@ local_only = "present" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars={"shared": "cli", "cli_only": "present"}, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -672,14 +677,14 @@ repo_files=base_repo_files() | { "models/orders.sql": ( - "MODEL (materialized incremental, incremental_strategy append, " + "MODEL (description 'Test model orders.', materialized incremental, incremental_strategy append, " "full_refresh false);\n\nSELECT 1 AS id\n" ), }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -704,16 +709,16 @@ 'name = "demo"\nadapter = "duckdb"\n\n[defaults]\nfull_refresh = false\n' ), "models/a_incremental.sql": ( - "MODEL (materialized incremental, incremental_strategy append);" + "MODEL (description 'Test model a_incremental.', materialized incremental, incremental_strategy append);" "\n\nSELECT 1 AS id\n" ), - "models/b_view.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - "models/c_table.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", + "models/b_view.sql": "MODEL (description 'Test model b_view.', materialized view);\n\nSELECT 1 AS id\n", + "models/c_table.sql": "MODEL (description 'Test model c_table.', materialized table);\n\nSELECT 1 AS id\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, None, None), + expected_model_schema_names=("a_incremental", "b_view", "c_table"), expected_model_config_values=( { "full_refresh": False, @@ -746,12 +751,12 @@ '[defaults]\nmaterialized = "${model_kind}"\nfull_refresh = false\n' 'incremental_strategy = "append"\n' ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS id\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -800,7 +805,7 @@ """.strip() + "\n", "models/staging/orders.sql": ( - "MODEL (\n unique_key [order_id]," + "MODEL (description 'Test model orders.',\n unique_key [order_id]," "\n cursor event_time," "\n cursor_type timestamp," "\n cursor_grain second," @@ -810,7 +815,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -859,7 +864,7 @@ """.strip() + "\n", "models/staging/orders.sql": ( - "MODEL (\n cursor event_time," + "MODEL (description 'Test model orders.',\n cursor event_time," "\n cursor_type timestamp," "\n cursor_grain second," "\n);\n\nselect 1\n" @@ -868,7 +873,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -917,7 +922,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", row_diff_exclude_columns [run_id, loaded_at], row_diff_tolerances ( by_type ( @@ -941,7 +946,7 @@ selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "row_diff_exclude_columns": ("loaded_at", "run_id"), @@ -1001,7 +1006,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, incremental_strategy append, alias '${CTX:model.name}_${CTX:run.target}', @@ -1023,7 +1028,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "dev_analytics_kevin", @@ -1078,7 +1083,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias 'orders_${CTX:run.target}', config ( run_label '${CTX:run.id}', @@ -1093,7 +1098,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "db_ci", @@ -1127,13 +1132,13 @@ schema = "${CTX:model.name}_schema" """.strip() + "\n", - "models/orders.sql": "MODEL ();\n\nselect 1\n", - "models/customers.sql": "MODEL ();\n\nselect 2\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", + "models/customers.sql": "MODEL (description 'Test model customers.');\n\nselect 2\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, None), + expected_model_schema_names=("customers", "orders"), expected_model_config_values=( {"schema": "customers_schema"}, {"schema": "orders_schema"}, @@ -1165,7 +1170,7 @@ """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", cursor event_time, cursor_type timestamp, cursor_grain second, @@ -1179,7 +1184,7 @@ selected_target=None, cli_vars=None, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "materialized": "incremental", @@ -1316,12 +1321,12 @@ schema = "preserve" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "analytics_team_prod", @@ -1381,7 +1386,7 @@ def grant_select(target: str) -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias orders_dev, post_hooks [inline_sql('@grant_select("@@CTX:destination.qualified")')], ); @@ -1412,6 +1417,7 @@ def grant_select(target: str) -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. table: orders """.strip() + "\n", @@ -1423,7 +1429,7 @@ def grant_select(target: str) -> str: "optional_suffix": None, }, run_id="run_123", - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "schema": "marts", @@ -1504,7 +1510,7 @@ def grant_select(target: str) -> str: } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -1525,7 +1531,7 @@ def grant_select(target: str) -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_path_defaults=(None,), expected_seed_names=(), @@ -1583,7 +1589,7 @@ def source_columns() -> str: """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [inline_sql("insert into @@audit_schema.load_log select '@@ENV:USER_NAME'")], columns (order_id (audits [source_filter])), ); @@ -1595,7 +1601,7 @@ def source_columns() -> str: """.strip() + "\n", "functions/sql/is_large_order.sql": """ -FUNCTION ( +FUNCTION (description "Test function is_large_order.", arguments (amount INTEGER), returns BOOLEAN, ); @@ -1635,10 +1641,12 @@ def source_columns() -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. expression: | SELECT @source_columns(), '@@ENV:USER_NAME' AS loaded_by FROM @@raw_schema.raw_orders - name: raw_table + description: Test source raw_table. database: ${raw_database} schema: ${raw_schema} table: orders_${ENV:USER_NAME} @@ -1747,7 +1755,7 @@ def source_columns() -> str: """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", alias orders_dev, post_hooks [ inline_sql("select '@@CTX:destination.qualified' as target"), @@ -1765,6 +1773,7 @@ def source_columns() -> str: @hook def notify(ctx, message): + '''Test hook notify.''' return None """.strip() + "\n", @@ -1772,7 +1781,7 @@ def notify(ctx, message): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "schema": "marts", @@ -1816,7 +1825,7 @@ def notify(ctx, message): """.strip() + "\n", "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", post_hooks [sql("record_access", role: "O'Brien")], ); @@ -1834,7 +1843,7 @@ def notify(ctx, message): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "database": "analytics", @@ -1873,7 +1882,7 @@ def notify(ctx, message): repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", pre_hooks [ python("defaulted", channel: "alerts"), python("flexible", extra: "value"), @@ -1891,21 +1900,25 @@ def notify(ctx, message): @hook def defaulted(context, channel, message="ready"): + '''Test hook defaulted.''' return None @hook def flexible(ctx, **kwargs): + '''Test hook flexible.''' return None @hook def context_named(hook_context, message): + '''Test hook context_named.''' return None @hook def optional_positional_only(ctx, channel="alerts", /): + '''Test hook optional_positional_only.''' return None """.strip() + "\n", @@ -1913,7 +1926,7 @@ def optional_positional_only(ctx, channel="alerts", /): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "pre_hooks": [ @@ -1938,7 +1951,7 @@ def optional_positional_only(ctx, channel="alerts", /): repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", post_hooks [ python("notify", message: "@@CTX:destination.qualified", macro_text: "@format_message()"), ], @@ -1953,6 +1966,7 @@ def optional_positional_only(ctx, channel="alerts", /): @hook def notify(ctx, message, macro_text): + '''Test hook notify.''' return None """.strip() + "\n", @@ -1960,7 +1974,7 @@ def notify(ctx, message, macro_text): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=( { "post_hooks": [ @@ -2013,7 +2027,7 @@ def status_match(column_name: str, status: str) -> str: """.strip() + "\n", "functions/sql/is_completed_order.sql": """ -FUNCTION ( +FUNCTION (description "Test function is_completed_order.", database ${udf_database}, schema ${udf_schema}, arguments (order_status ${status_type}), @@ -2074,10 +2088,10 @@ def status_match(column_name: str, status: str) -> str: """.strip() + "\n", "models/fact_orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(7)\n' + 'MODEL (description "Test model fact_orders.");\n\nSELECT * FROM __table_fn("customer_orders")(7)\n' ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -2094,7 +2108,7 @@ def status_match(column_name: str, status: str) -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("fact_orders",), expected_model_config_values=({},), expected_model_query_sqls=('SELECT * FROM __table_fn("customer_orders")(7)',), expected_model_path_defaults=(None,), @@ -2131,7 +2145,7 @@ def status_match(column_name: str, status: str) -> str: | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", returns table ( order_id INTEGER, status VARCHAR @@ -2188,7 +2202,7 @@ def project_columns() -> str: return "order_id" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (name "first"); @@ -2226,8 +2240,10 @@ def project_columns() -> str: "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. table: orders - name: raw_customers + description: Test source raw_customers. table: customers """.strip() + "\n", @@ -2235,7 +2251,7 @@ def project_columns() -> str: selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2316,6 +2332,7 @@ def project_columns() -> str: packages=["faker"], ) def main(order_status): + '''Test function main.''' return order_status == "completed" """.strip() + "\n", @@ -2336,6 +2353,7 @@ def main(order_status): expected_sql_function_body_sqls=( """ def main(order_status): + \"\"\"Test function main.\"\"\" return order_status == 'completed' """.strip(), ), @@ -2362,7 +2380,7 @@ def main(order_status): repo_files=base_repo_files() | { "models/marts/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", audits [model_not_null], columns ( order_id (audits [column_not_null]), @@ -2375,6 +2393,7 @@ def main(order_status): "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - source_not_null columns: @@ -2440,7 +2459,7 @@ def main(order_status): repo_files=base_repo_files() | { "models/marts/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (nullable false, audits [not_null, unique]), status (audits [accepted_values (values ["placed", "completed"])]), @@ -2454,10 +2473,11 @@ def main(order_status): select 1 as order_id, 'placed' as status, null as customer_id """.strip() + "\n", - "models/marts/customers.sql": "MODEL ();\n\nselect 1 as customer_id\n", + "models/marts/customers.sql": "MODEL (description 'Test model customers.');\n\nselect 1 as customer_id\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. columns: - name: order_id audits: @@ -2468,7 +2488,7 @@ def main(order_status): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None, "orders"), + expected_model_schema_names=("customers", "orders"), expected_model_config_values=({}, {}), expected_model_query_sqls=( "select 1 as customer_id", @@ -2509,7 +2529,7 @@ def main(order_status): description="skips generic audit definitions as direct executable audits", repo_files=base_repo_files() | { - "models/marts/orders.sql": "MODEL ();\n\nselect 1\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "audits/generic/custom_check.sql": """ AUDIT (); @@ -2520,7 +2540,7 @@ def main(order_status): selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2538,13 +2558,13 @@ def main(order_status): description="warns when project generic audit shadows built-in audit", repo_files=base_repo_files() | { - "models/marts/orders.sql": "MODEL ();\n\nselect 1\n", + "models/marts/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "audits/generic/not_null.sql": "AUDIT ();\n\nSELECT 1\n", }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2564,11 +2584,14 @@ def main(order_status): ), BuildCompileInputsTestCase( description="generates clickstate style run ids when none are provided", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target=None, cli_vars=None, run_id=None, - expected_model_schema_names=(None,), + expected_model_schema_names=("orders",), expected_model_config_values=({},), expected_model_query_sqls=("select 1",), expected_model_path_defaults=(None,), @@ -2592,12 +2615,12 @@ def main(order_status): sql_validation = false """.strip() + "\n", - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2623,12 +2646,12 @@ def main(order_status): sql_analysis = false """.strip() + "\n", - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2646,13 +2669,13 @@ def main(order_status): description="allows invalid sql when no_sql_validation flag is set", repo_files=base_repo_files() | { - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, cli_vars=None, run_id="test_run", no_sql_validation=True, - expected_model_schema_names=(None,), + expected_model_schema_names=("broken",), expected_model_config_values=({},), expected_model_query_sqls=("SELEC id FROM (SELECT 1",), expected_model_path_defaults=(None,), @@ -2690,14 +2713,14 @@ def main(order_status): """.strip() + "\n", "models/fact_orders.sql": ( - 'MODEL ();\n\nselect * from __dbt_ref("stg_orders") ' + 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("stg_orders") ' 'union all select * from __dbt_ref("stripe", "orders")\n' ), }, selected_target=None, cli_vars=None, run_id="test_run", - expected_model_schema_names=(None,), + expected_model_schema_names=("fact_orders",), expected_model_config_values=({},), expected_model_query_sqls=( 'select * from __dbt_ref("stg_orders") ' @@ -3056,9 +3079,9 @@ def test_given_preserved_target_when_building_functions_then_logical_namespaces_ """.strip() + "\n", "functions/sql/defaulted.sql": ( - "FUNCTION (returns INTEGER, tags [reference, daily]);\n\n1\n" + "FUNCTION (description 'Test function defaulted.', returns INTEGER, tags [reference, daily]);\n\n1\n" ), - "models/model_defaulted.sql": "MODEL ();\n\nSELECT 1 AS id\n", + "models/model_defaulted.sql": "MODEL (description 'Test model model_defaulted.');\n\nSELECT 1 AS id\n", "functions/python/explicit.py": """ from sqlbuild.functions import udf @@ -3070,6 +3093,7 @@ def test_given_preserved_target_when_building_functions_then_logical_namespaces_ schema="function_schema", ) def main(): + '''Test function main.''' return 1 """.strip() + "\n", @@ -3078,6 +3102,7 @@ def main(): @udf(arguments={}, returns="INTEGER", runtime_version="3.11") def main(): + '''Test function main.''' return 1 """.strip() + "\n", @@ -3219,7 +3244,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov """.strip() + "\n", "models/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __table_fn("missing_orders")(1) """.strip() @@ -3235,7 +3260,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER, customer_id INTEGER) ); @@ -3253,9 +3278,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', - "models/orders.sql": ('MODEL ();\n\nSELECT * FROM __table_fn("order_total")(1)\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("order_total")(1)\n' + ), "functions/sql/order_total.sql": """ -FUNCTION ( +FUNCTION (description "Test function order_total.", arguments (order_id INTEGER), returns INTEGER ); @@ -3275,9 +3302,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sqlbuild_project.toml": ( 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), - "models/orders.sql": ('MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")\n'), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")\n' + ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3298,7 +3327,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - "MODEL ();\n\nSELECT * FROM __table_fn('customer_orders')(1)\n" + "MODEL (description 'Test model orders.');\n\nSELECT * FROM __table_fn('customer_orders')(1)\n" ), }, selected_target=None, @@ -3313,10 +3342,10 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(1, 2)\n' + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")(1, 2)\n' ), "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3337,7 +3366,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT * FROM __table_fn("customer_orders")(1,)\n' + 'MODEL (description "Test model orders.");\n\nSELECT * FROM __table_fn("customer_orders")(1,)\n' ), }, selected_target=None, @@ -3350,7 +3379,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov | { "sqlbuild_project.toml": 'name = "demo"\nadapter = "duckdb"\n', "functions/sql/customer_orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function customer_orders.", arguments (customer_id INTEGER), returns table (order_id INTEGER) ); @@ -3379,7 +3408,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __source("missing_source") """.strip() @@ -3394,7 +3423,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", materialized incremental, contract enforced, columns ( @@ -3419,7 +3448,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -3444,7 +3473,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -3485,7 +3514,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL (); +MODEL (description "Test model orders."); SELECT * FROM __dbt_ref("stg_orders") """.strip() @@ -3503,7 +3532,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "functions/sql/orders.sql": """ -FUNCTION ( +FUNCTION (description "Test function orders.", returns table (order_id INTEGER) ); @@ -3543,6 +3572,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - source_not_null: source: other_source @@ -3564,7 +3594,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov repo_files=base_repo_files() | { "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", columns ( order_id (nullable true, audits [not_null]), ), @@ -3587,6 +3617,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. audits: - missing_definition """.strip() @@ -3615,7 +3646,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body lacks top level test ctes", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3631,10 +3662,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body lacks ceremonial select one", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3659,7 +3691,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown source mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3682,7 +3714,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown seed mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "tests/unit/orders.sql": """ TEST (); @@ -3722,10 +3754,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw__orders + description: Test source raw__orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3766,11 +3799,12 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov } """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "seeds/countries.csv": "country_code,country_name\nUS,United States\n", "seeds/schema.yml": """ seeds: - name: countries + description: Test seed countries. columns: - name: country_code type: VARCHAR @@ -3800,10 +3834,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body references an unknown macro mock", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3834,6 +3869,7 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3858,10 +3894,11 @@ def test_given_project_and_local_environment_when_resolving_then_local_values_ov description="raises when a compiled test body uses a reserved helper cte name", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", "sources/raw.yml": """ sources: - name: raw_orders + description: Test source raw_orders. """.strip() + "\n", "tests/unit/orders.sql": """ @@ -3914,7 +3951,7 @@ def project_columns() -> str: return "customer_id" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -3925,7 +3962,7 @@ def project_columns() -> str: repo_files=base_repo_files() | { "macros/orders.py": "def DailyOrders():\n return 'order_id'\n", - "models/staging/orders.sql": "MODEL ();\n\nselect @DailyOrders()\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect @DailyOrders()\n", }, selected_target=None, run_id=None, @@ -3944,7 +3981,7 @@ def dynamic_schema() -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", schema '@dynamic_schema()', ); @@ -3958,7 +3995,10 @@ def dynamic_schema() -> str: ), BuildCompileInputsErrorTestCase( description="raises when the selected target does not exist", - repo_files=base_repo_files() | {"models/staging/orders.sql": "MODEL ();\n\nselect 1\n"}, + repo_files=base_repo_files() + | { + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n" + }, selected_target="missing", run_id=None, expected_error_fragment="Unknown target 'missing'", @@ -3972,7 +4012,7 @@ def dynamic_schema() -> str: target = "missing" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -3989,7 +4029,7 @@ def dynamic_schema() -> str: default_target = "missing" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4008,7 +4048,7 @@ def dynamic_schema() -> str: user = "${missing}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4026,7 +4066,7 @@ def dynamic_schema() -> str: path = "${ENV:SQLBUILD_DB_PATH}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4044,7 +4084,7 @@ def dynamic_schema() -> str: """.strip() + "\n", "models/staging/orders.sql": """ -MODEL ( +MODEL (description "Test model orders.", schema '${CTX:this}', ); @@ -4074,7 +4114,7 @@ def dynamic_schema() -> str: schema = "dev_${CTX:destination.missing}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4097,7 +4137,7 @@ def dynamic_schema() -> str: database = "dev_${CTX:model.database}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4118,7 +4158,7 @@ def dynamic_schema() -> str: second = "${first}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4141,7 +4181,7 @@ def dynamic_schema() -> str: schema = "${SQLBUILD:user}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4159,7 +4199,7 @@ def dynamic_schema() -> str: path = "${CTX:schema}" """.strip() + "\n", - "models/staging/orders.sql": "MODEL ();\n\nselect 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4169,7 +4209,7 @@ def dynamic_schema() -> str: description="raises when model query sql has invalid syntax", repo_files=base_repo_files() | { - "models/staging/broken.sql": "MODEL ();\n\nSELEC id FROM (SELECT 1\n", + "models/staging/broken.sql": "MODEL (description 'Test model broken.');\n\nSELEC id FROM (SELECT 1\n", }, selected_target=None, run_id=None, @@ -4180,7 +4220,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks ['SELECT 1']);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model broken.', post_hooks ['SELECT 1']);\n\nSELECT 1 AS id\n" ), }, selected_target=None, @@ -4192,7 +4232,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks [inline_sql('')]);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model broken.', post_hooks [inline_sql('')]);\n\nSELECT 1 AS id\n" ), }, selected_target=None, @@ -4209,9 +4249,9 @@ def dynamic_schema() -> str: 'name = "demo"\nadapter = "duckdb"\n\n[settings]\nsql_validation = false\n' ), "models/staging/broken.sql": ( - 'MODEL (post_hooks [sql("dynamic", statement: "")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [sql("dynamic", statement: "")]);\n\nSELECT 1 AS id\n' ), - "hooks/sql/dynamic.sql": "HOOK ();\n\n@statement\n", + "hooks/sql/dynamic.sql": "HOOK (description 'Test hook dynamic.');\n\n@statement\n", }, selected_target=None, run_id=None, @@ -4224,7 +4264,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - "MODEL (post_hooks [" + "MODEL (description 'Test model broken.', post_hooks [" "inline_sql('GRANT SELECT ON ${CTX:destination.qualified} TO analyst')" "]);\n\n" "SELECT 1 AS id\n" @@ -4242,7 +4282,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), }, selected_target=None, @@ -4256,7 +4296,7 @@ def dynamic_schema() -> str: repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("known"), python("missing")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("known"), python("missing")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4264,6 +4304,7 @@ def dynamic_schema() -> str: @hook def known(ctx): + '''Test hook known.''' return None """.strip() + "\n", @@ -4279,7 +4320,7 @@ def known(ctx): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (pre_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", pre_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4287,6 +4328,7 @@ def known(ctx): @hook def notify(ctx, channel): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4302,7 +4344,7 @@ def notify(ctx, channel): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify", unknown: "value")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify", unknown: "value")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4310,6 +4352,7 @@ def notify(ctx, channel): @hook def notify(ctx): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4325,7 +4368,7 @@ def notify(ctx): repo_files=base_repo_files() | { "models/staging/broken.sql": ( - 'MODEL (post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model broken.", post_hooks [python("notify")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from sqlbuild.hooks import hook @@ -4333,6 +4376,7 @@ def notify(ctx): @hook def notify(ctx, channel, /): + '''Test hook notify.''' return None """.strip() + "\n", @@ -4348,7 +4392,7 @@ def notify(ctx, channel, /): description="raises when model header tags is a string instead of list", repo_files=base_repo_files() | { - "models/staging/orders.sql": "MODEL (tags nightly);\n\nSELECT 1\n", + "models/staging/orders.sql": "MODEL (description 'Test model orders.', tags nightly);\n\nSELECT 1\n", }, selected_target=None, run_id=None, @@ -4358,7 +4402,7 @@ def notify(ctx, channel, /): description="raises when dbt ref has no discovered manifest", repo_files=base_repo_files() | { - "models/fact_orders.sql": 'MODEL ();\n\nselect * from __dbt_ref("orders")\n', + "models/fact_orders.sql": 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("orders")\n', }, selected_target=None, run_id=None, @@ -4389,7 +4433,7 @@ def notify(ctx, channel, /): } """.strip() + "\n", - "models/fact_orders.sql": 'MODEL ();\n\nselect * from __dbt_ref("orders")\n', + "models/fact_orders.sql": 'MODEL (description "Test model fact_orders.");\n\nselect * from __dbt_ref("orders")\n', }, selected_target=None, run_id=None, @@ -4413,7 +4457,7 @@ def notify(ctx, channel, /): } """.strip() + "\n", - "models/orders.sql": "MODEL ();\n\nselect 1\n", + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nselect 1\n", }, selected_target=None, run_id=None, @@ -4457,7 +4501,7 @@ def test_given_attachment_conflicts_when_building_compile_inputs_then_it_raises_ repo_files=base_repo_files() | { "models/staging/orders.sql": ( - 'MODEL (post_hooks [python("notify", channel: "alerts")]);\n\nSELECT 1 AS id\n' + 'MODEL (description "Test model orders.", post_hooks [python("notify", channel: "alerts")]);\n\nSELECT 1 AS id\n' ), "hooks/python/notifications.py": """ from pathlib import Path @@ -4466,6 +4510,7 @@ def test_given_attachment_conflicts_when_building_compile_inputs_then_it_raises_ @hook def notify(ctx, channel): + '''Test hook notify.''' Path(__file__).with_name("executed.marker").write_text("executed", encoding="utf-8") """.strip() + "\n", @@ -4648,7 +4693,7 @@ def test_given_retention_layers_when_building_model_config_then_precedence_is_re ) ), "models/orders.sql": ( - "MODEL (materialized table, time_travel_retention 30d);\n\nSELECT 1 AS id\n" + "MODEL (description 'Test model orders.', materialized table, time_travel_retention 30d);\n\nSELECT 1 AS id\n" ), }, expected_desired_days=30, @@ -4685,7 +4730,7 @@ def test_given_authored_retention_header_when_building_inputs_then_typed_policy_ repo_files=base_repo_files() | { "models/customer_snapshot.sql": """ -MODEL ( +MODEL (description "Test model customer_snapshot.", materialized snapshot, contract enforced, columns ( @@ -4741,7 +4786,7 @@ def test_given_snapshot_contract_schema_change_conflict_when_building_then_error "seeds/waffle_types.csv": "waffle_type_id,waffle_name\n1,Classic\n", "seeds/schema.yml": ( "seeds:\n" - " - name: waffle_types\n" + " - name: waffle_types\n description: Test seed waffle_types.\n" " columns:\n" " - name: waffle_type_id\n" " type: INTEGER\n" @@ -4749,7 +4794,7 @@ def test_given_snapshot_contract_schema_change_conflict_when_building_then_error " type: VARCHAR\n" ), "models/orders.sql": ( - 'MODEL ();\n\nSELECT waffle_type_id FROM __seed("waffle_types")' + 'MODEL (description "Test model orders.");\n\nSELECT waffle_type_id FROM __seed("waffle_types")' ), }, expected_model_count=1, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index fd95957051..d162c1b057 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -28,7 +28,7 @@ description="resolved connection override becomes the effective connection", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, resolved_connection={"database": "/abs/warehouse.duckdb", "schema": "analytics"}, expected_effective_connection={ @@ -40,7 +40,7 @@ description="without override the merged project connection is used", repo_files={ "sqlbuild_project.toml": _PROJECT_TOML, - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", + "models/orders.sql": "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id", }, resolved_connection=None, expected_effective_connection={"database": "warehouse.duckdb"}, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 914534f838..7738441e93 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -61,7 +61,7 @@ def test_given_nested_type_key_when_locating_declared_type_then_returns_top_leve def test_given_dynamic_family_when_generating_contract_then_does_not_freeze_runtime_members( test_case: DynamicContractAdoptionTestCase, ) -> None: - authored_sql: str = """MODEL ( + authored_sql: str = """MODEL (description "Test model.", contract enforced, columns (customer_id (type INTEGER)), dynamic_columns ( diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py index 18b847af75..f6bc24dc9d 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py @@ -353,7 +353,7 @@ def test_given_deferred_output_locations_when_contract_fails_then_locates_author type_enforcement=True, contract="enforced", authored_sql=( - "MODEL (columns (order_id (type INTEGER)));\n" + "MODEL (description 'Test model.', columns (order_id (type INTEGER)));\n" "SELECT\n" " 1 AS order_id,\n" " 2 AS amount\n" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index b8a1d47041..0425fc326c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -620,6 +620,7 @@ class ParseSourcesYamlDltTestCase: expected_dlt_names: tuple[str, ...] expected_schemas: tuple[str | None, ...] expected_destination_config: dict[str, object] = field(default_factory=dict) + expected_descriptions: tuple[str | None, ...] = () @dataclass(frozen=True) @@ -760,8 +761,9 @@ class DiscoverLoaderFunctionsTestCase: description: str files: dict[str, str] expected_names: tuple[str, ...] - expected_targets: tuple[str | None, ...] - expected_dependency_counts: tuple[int, ...] + expected_targets: tuple[str | None, ...] = () + expected_dependency_counts: tuple[int, ...] = () + expected_descriptions: tuple[str | None, ...] = () expected_write_strategies: tuple[str | None, ...] = () expected_cursor_columns: tuple[str | None, ...] = () expected_unique_keys: tuple[tuple[str, ...], ...] = () @@ -928,6 +930,7 @@ class FunctionHeaderKeysTestCase: contents: str expected_error: str = "" expected_keys: tuple[str, ...] = () + expected_description: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index 8acb9febbe..560c3393e6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -227,3 +227,53 @@ def test_given_loader_import_error_when_discovering_loaders_then_raises_clear_er with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): discover_python_node_functions(project_dir=tmp_path) + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoverLoaderFunctionsTestCase( + description="loader descriptions come from the decorator or the docstring", + files={ + "python/loaders/orders.py": ''' +from sqlbuild.loaders import loader + +@loader +def raw_orders(ctx): + """Orders from the storefront export.""" + return [] + +@loader(description="Refunds from the payments export") +def raw_refunds(ctx): + """Ignored because the decorator wins.""" + return [] + +@loader +def raw_returns(ctx): + return [] +''', + }, + expected_names=("raw_orders", "raw_refunds", "raw_returns"), + expected_descriptions=( + "Orders from the storefront export.", + "Refunds from the payments export", + None, + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_loader_descriptions_when_discovering_then_decorator_wins_over_docstring( + test_case: DiscoverLoaderFunctionsTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + file_path: Path = tmp_path / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(contents, encoding="utf-8") + + result: tuple[DiscoveredLoaderFunction, ...] = tuple( + sorted(discover_python_node_functions(project_dir=tmp_path).loaders, key=lambda x: x.name) + ) + + assert tuple(loader.name for loader in result) == test_case.expected_names + assert tuple(loader.description for loader in result) == test_case.expected_descriptions diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py index 2c9d80068c..d6b6e26573 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_function_header_keys.py @@ -21,10 +21,11 @@ description="sql udf with every supported key", file_name="normalize_status.sql", contents=( - "FUNCTION (\n arguments (status VARCHAR),\n returns VARCHAR,\n" + 'FUNCTION (\n description "Normalise an order status",\n' + " arguments (status VARCHAR),\n returns VARCHAR,\n" " database analytics,\n schema udfs,\n tags [orders],\n);\n\nlower(status)\n" ), - expected_keys=("arguments", "database", "returns", "schema", "tags"), + expected_keys=("arguments", "database", "description", "returns", "schema", "tags"), ), FunctionHeaderKeysTestCase( description="table function with a returns table declaration", @@ -68,11 +69,10 @@ def test_given_supported_function_header_keys_when_parsing_then_accepts_header( file_name="expand_order.sql", contents=( 'FUNCTION (\n description "Expand one order",\n arguments (order_id INTEGER),\n' - " returns table (order_id INTEGER),\n owner orders_team,\n);\n\nSELECT order_id\n" - ), - expected_error=( - "FUNCTION() in 'expand_order.sql:2' has unsupported keys: description, owner" + " returns table (order_id INTEGER),\n owner orders_team,\n team sales,\n);\n\n" + "SELECT order_id\n" ), + expected_error=("FUNCTION() in 'expand_order.sql:5' has unsupported keys: owner, team"), ), ], ids=lambda case: case.description, @@ -114,5 +114,55 @@ def test_given_unknown_python_udf_key_when_parsing_then_rejects_key_with_locatio assert raised.value.message == test_case.expected_error +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderKeysTestCase( + description="docstring describes the udf", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN")\n' + 'def main(amount):\n """Whether an order amount is large."""\n' + " return amount > 100\n" + ), + expected_description="Whether an order amount is large.", + ), + FunctionHeaderKeysTestCase( + description="decorator description wins over the docstring", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(description="Large order flag", arguments={"amount": "INTEGER"}, ' + 'returns="BOOLEAN")\n' + 'def main(amount):\n """Whether an order amount is large."""\n' + " return amount > 100\n" + ), + expected_description="Large order flag", + ), + FunctionHeaderKeysTestCase( + description="undocumented udf has no description", + file_name="is_large_order_py.py", + contents=( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN")\n' + "def main(amount):\n return amount > 100\n" + ), + expected_description=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_udf_when_parsing_then_description_comes_from_decorator_or_docstring( + test_case: FunctionHeaderKeysTestCase, +) -> None: + header_values: dict[str, object] + header_values, _, _ = parse_python_function( + contents=test_case.contents, file_path=Path(test_case.file_name) + ) + + assert header_values.get("description") == test_case.expected_description + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 8ff38e7023..e68bbc6dc7 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -483,6 +483,7 @@ def test_given_ingestr_source_yaml_when_parsing_then_stores_typed_integration_co schema: main resources: - name: raw_orders + description: Orders from the storefront database table: orders write_disposition: merge primary_key: order_id @@ -497,6 +498,7 @@ def test_given_ingestr_source_yaml_when_parsing_then_stores_typed_integration_co expected_dlt_names=("orders", "customers"), expected_schemas=("raw", "raw_customers_override"), expected_destination_config={"naming_convention": "sql_cs_v1", "create_indexes": True}, + expected_descriptions=("Orders from the storefront database", None), ) ], ids=lambda case: case.description, @@ -526,6 +528,8 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( tuple(config.resource.dlt_name for config in typed_configs) == test_case.expected_dlt_names ) assert tuple(entry.schema for entry in source_entries) == test_case.expected_schemas + assert tuple(entry.description for entry in source_entries) == test_case.expected_descriptions + assert all("description" not in config.resource.raw_config for config in typed_configs) assert tuple(config.resource.schema for config in typed_configs) == test_case.expected_schemas assert tuple(config.destination for config in typed_configs) == ( test_case.expected_destination_config, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 73797d57f1..77a59535d7 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -958,7 +958,7 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc DiscoveredSqlHookFile( file_path=Path("/project/hooks/sql/record_access.sql"), relative_path=Path("hooks/sql/record_access.sql"), - contents="HOOK ();\n\nSELECT @'role'\n", + contents="HOOK (description 'Test hook record_access.');\n\nSELECT @'role'\n", header_values={"description": "Record access"}, sql_body="SELECT @'role'", name="record_access", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index ffb78f706f..880fecbf85 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -54,8 +54,8 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ tmp_path, { "sqlbuild_project.toml": 'name = "source_batch"\nadapter = "duckdb"\n', - "sources/orders.yml": "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n", - "models/combined.sql": 'MODEL ();\nSELECT o.id FROM __source("orders") o JOIN __source("customers") c ON o.id = c.id', + "sources/orders.yml": "sources:\n - name: orders\n description: Test source orders.\n table: orders\n - name: customers\n description: Test source customers.\n table: customers\n", + "models/combined.sql": 'MODEL (description "Test model combined.");\nSELECT o.id FROM __source("orders") o JOIN __source("customers") c ON o.id = c.id', }, ) adapter: DuckDbAdapter = DuckDbAdapter() @@ -109,10 +109,10 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found { "sqlbuild_project.toml": 'name = "source_case"\nadapter = "duckdb"\n', "sources/orders.yml": ( - "sources:\n - name: orders\n" + "sources:\n - name: orders\n description: Test source orders.\n" f" schema: {test_case.declared_schema}\n table: orders\n" ), - "models/open_orders.sql": 'MODEL ();\nSELECT id FROM __source("orders")', + "models/open_orders.sql": 'MODEL (description "Test model open_orders.");\nSELECT id FROM __source("orders")', }, ) adapter: DuckDbAdapter = DuckDbAdapter() diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index 719197ed47..62a7809ca1 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -323,6 +323,7 @@ def audit_message(): @hook(name="before_orders") def before_orders(ctx): + '''Test hook before_orders.''' return audit_message() """.strip() + "\n", diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 5e62b7decd..e420ec69c1 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -41,7 +41,7 @@ from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig SCOPE_CACHE_PROJECT: str = 'name = "demo"\nadapter = "duckdb"\n' -SCOPE_CACHE_MODEL: str = "MODEL();\nSELECT 1 AS id\n" +SCOPE_CACHE_MODEL: str = "MODEL(description 'Test model.');\nSELECT 1 AS id\n" def unused_declaration_index(*, enforce_placement: bool) -> ScopeIndex: @@ -145,7 +145,7 @@ def discovered_model( return DiscoveredSqlModelFile( file_path=relative, relative_path=relative, - contents="MODEL(); SELECT 1", + contents="MODEL(description 'Test model.'); SELECT 1", header_values={"name": name}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index 33fb48fe73..d7167b9f2b 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -159,7 +159,11 @@ def test_given_no_cache_when_loading_twice_then_cache_is_bypassed( "test_case", ( FingerprintMutationCase( - "model content", "models/orders.sql", _MODEL, "MODEL();\nSELECT 2 AS id\n", True + "model content", + "models/orders.sql", + _MODEL, + "MODEL(description 'Test model orders.');\nSELECT 2 AS id\n", + True, ), FingerprintMutationCase( "declaration rendering", @@ -314,7 +318,7 @@ def test_given_target_aware_python_composition_when_switching_target_then_depend _PROJECT + '\ndefault_target = "dev"\n[targets.dev]\n[targets.prod]\n' ), "sqlbuild_local.toml": 'target = "dev"\n', - "models/orders.sql": "MODEL();\nSELECT @choose() AS value\n", + "models/orders.sql": "MODEL(description 'Test model orders.');\nSELECT @choose() AS value\n", "macros/choose.py": ( "from macros.dev import dev_macro\n" "from macros.prod import prod_macro\n\n" @@ -535,7 +539,7 @@ def test_given_placement_invalid_project_when_loading_then_complete_diagnostics_ { "sqlbuild_project.toml": _PROJECT, "models/domain/constants/value.sql": "CONSTANT (name value, value 1);", - "models/domain/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/domain/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ) @@ -563,7 +567,7 @@ def test_given_cached_placement_error_when_disabling_enforcement_then_cache_retu { "sqlbuild_project.toml": _PROJECT + "\n[scopes]\nenforce_placement = true\n", "constants/value.sql": "CONSTANT (name value, value 1);", - "models/orders.sql": 'MODEL ();\nSELECT @const("value") AS value', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("value") AS value', }, ) enforced: ScopeIndex = load_or_build_scope_index(project_dir=tmp_path) @@ -626,7 +630,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="SQL functions retain valid siblings", files={ - "functions/sql/valid.sql": "FUNCTION (returns INTEGER);\n1", + "functions/sql/valid.sql": "FUNCTION (description 'Test function valid.', returns INTEGER);\n1", "functions/sql/broken.sql": "FUNCTION (\nSELECT 1", }, expected_kind=ResourceKind.FUNCTION, @@ -635,7 +639,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="SQL hooks retain valid siblings", files={ - "hooks/sql/valid.sql": "HOOK ();\nSELECT 1", + "hooks/sql/valid.sql": "HOOK (description 'Test hook valid.');\nSELECT 1", "hooks/sql/broken.sql": "HOOK (\nSELECT 1", }, expected_kind=DeclarationKind.SQL_HOOK, @@ -653,7 +657,7 @@ def test_given_one_broken_model_when_loading_then_valid_and_broken_paths_remain_ TolerantCategoryCase( description="sources retain valid siblings", files={ - "sources/valid.yml": "sources:\n - name: raw_orders\n table: orders\n", + "sources/valid.yml": "sources:\n - name: raw_orders\n description: Test source raw_orders.\n table: orders\n", "sources/broken.yml": "sources: [\n", }, expected_kind=ResourceKind.SOURCE, diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index 6702a216b9..104d32d16c 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -15,6 +15,7 @@ from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.errors.setting_help.main.setting_note import setting_note +from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from tests.unit.src.sqlbuild.errors.setting_help.main._test_types import ( SettingCatalogueTestCase, SettingHelpTestCase, @@ -78,6 +79,17 @@ "to skip it, add this to the MODEL header:\n MODEL (\n" " sql_analysis false,\n ...\n );", ), + SettingHelpTestCase( + "a snippet help shows every line to write", + snippet_help( + purpose="to describe it", + target="add this to seeds/lookups.yml", + lines=("- name: product_types", " description: Product type catalogue"), + ), + "to describe it, add this to seeds/lookups.yml:\n" + " - name: product_types\n" + " description: Product type catalogue", + ), SettingHelpTestCase( "several helps keep one label each", join_helps("first", "second"), diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index 55b80c10a5..d0c43013f9 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -123,7 +123,7 @@ def build_clone_model_entry(*, schema: str, name: str) -> ModelPlanEntry: ), fingerprint_query_sql="SELECT 1", resolved_sql="SELECT 1", - logical_ddl="MODEL (materialized table);", + logical_ddl="MODEL (description 'Test model.', materialized table);", ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 9c576bb450..461c921660 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -265,7 +265,7 @@ def build_compiled_scenario(*, name: str) -> CompiledSqlScenario: scenario_file=DiscoveredSqlScenarioFile( file_path=Path(f"tests/scenarios/{name}.sql"), relative_path=Path(f"tests/scenarios/{name}.sql"), - contents="SCENARIO ();", + contents="SCENARIO (description 'Test scenario.');", header_values={}, sql_body="", name=name, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index efcfe9c8f6..dcd6964ca9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -423,7 +423,7 @@ def build_compiled_project_with_model_specs( model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( file_path=Path("/repo") / relative_path, relative_path=relative_path, - contents=f"MODEL ();\n\n{sql}\n", + contents=f"MODEL (description 'Test model.');\n\n{sql}\n", header_values={}, header_column_locations={}, output_column_locations={}, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py index 00131498a8..3ec89456f7 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_header_wrapping.py @@ -19,28 +19,28 @@ HeaderAuditWrapTestCase( "audits then their arguments go one per line", 50, - 'MODEL (\n audits [expression_is_true (name "positive", expression "total > 0")],\n);\n', - "MODEL (\n audits [\n expression_is_true (\n" + 'MODEL (\n description "Test model.",\n audits [expression_is_true (name "positive", expression "total > 0")],\n);\n', + 'MODEL (\n description "Test model.",\n audits [\n expression_is_true (\n' ' name "positive",\n expression "total > 0",\n ),\n ],\n);\n', ), HeaderAuditWrapTestCase( "quoted values are never split", 30, - 'MODEL (\n audits [expression_is_true (expression "a, b, c, d > 0")],\n);\n', - "MODEL (\n audits [\n expression_is_true (\n" + 'MODEL (\n description "Test model.",\n audits [expression_is_true (expression "a, b, c, d > 0")],\n);\n', + 'MODEL (\n description "Test model.",\n audits [\n expression_is_true (\n' ' expression "a, b, c, d > 0",\n ),\n ],\n);\n', ), HeaderAuditWrapTestCase( "a line with a comment is left alone", 30, - "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", - "MODEL (\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n", + 'MODEL (\n description "Test model.",\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n', + 'MODEL (\n description "Test model.",\n audits [not_null, unique, accepted_values (values [1, 2])], -- keep\n);\n', ), HeaderAuditWrapTestCase( "lists outside audits are left alone", 30, - "MODEL (\n tags [orders, customers, products, inventory],\n);\n", - "MODEL (\n tags [orders, customers, products, inventory],\n);\n", + 'MODEL (\n description "Test model.",\n tags [orders, customers, products, inventory],\n);\n', + 'MODEL (\n description "Test model.",\n tags [orders, customers, products, inventory],\n);\n', ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py index 8d39b52324..44546af55e 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_headers.py @@ -17,7 +17,7 @@ [ ScanHeadersTestCase( description="finds model header", - contents="MODEL (\n materialized table\n);\nSELECT 1\n", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\n', expected_kinds=("MODEL",), ), ScanHeadersTestCase( @@ -57,7 +57,7 @@ ScanHeadersTestCase( description="ignores header keyword used as a nested column name", contents=( - "MODEL (\n columns (\n object_id (type INTEGER),\n" + 'MODEL (\n description "Test model.",\n columns (\n object_id (type INTEGER),\n' " SCHEMA (type VARCHAR(100)),\n ),\n);\nSELECT 1 AS object_id\n" ), expected_kinds=("MODEL",), @@ -65,7 +65,7 @@ ScanHeadersTestCase( description="ignores header-like text inside comments", contents=( - '-- TEST ();\n/*\nMODEL (not a header);\n*/\nMODEL (description "ok");\nSELECT 1\n' + '-- TEST ();\n/*\nMODEL (description "Test model.", not a header);\n*/\nMODEL (description "ok");\nSELECT 1\n' ), expected_kinds=("MODEL",), ), @@ -86,7 +86,7 @@ ), ScanHeadersTestCase( description="standard SQL backslash does not escape closing quote", - contents="SCENARIO ();\nSELECT 'C:\\temp\\'\nTEST ();\nSELECT 1\n", + contents="SCENARIO (description \"Test scenario.\");\nSELECT 'C:\\temp\\'\nTEST ();\nSELECT 1\n", expected_kinds=("SCENARIO", "TEST"), ), ], @@ -104,7 +104,7 @@ def test_given_contents_when_scanning_headers_then_kinds_match_expected( [ ScanHeadersTestCase( description="first header only", - contents="MODEL ();\nSELECT 1\nTEST ();\nSELECT 2\n", + contents='MODEL (description "Test model.");\nSELECT 1\nTEST ();\nSELECT 2\n', expected_kinds=("MODEL",), ) ], @@ -125,13 +125,13 @@ def test_given_single_header_resource_when_scanning_first_only_then_body_is_not_ [ HeaderSpanTextTestCase( description="span covers header keyword through terminator", - contents="MODEL (\n materialized table\n);\nSELECT 1\n", - expected_span_text="MODEL (\n materialized table\n);", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\n', + expected_span_text='MODEL (\n description "Test model.",\n materialized table\n);', ), HeaderSpanTextTestCase( description="span without terminator ends at close paren", - contents="MODEL (\n materialized table\n)\nSELECT 1\n", - expected_span_text="MODEL (\n materialized table\n)", + contents='MODEL (\n description "Test model.",\n materialized table\n)\nSELECT 1\n', + expected_span_text='MODEL (\n description "Test model.",\n materialized table\n)', ), HeaderSpanTextTestCase( description="escaped quotes and quoted parens stay inside the span", @@ -165,7 +165,7 @@ def test_given_model_file_when_scanning_then_span_covers_header_and_terminator( [ SqlBodyRangesTestCase( description="bodies exclude headers and keep sql text", - contents="MODEL (\n materialized table\n);\nSELECT 1\nTEST ();\nSELECT 2\n", + contents='MODEL (\n description "Test model.",\n materialized table\n);\nSELECT 1\nTEST ();\nSELECT 2\n', expected_fragments=("SELECT 1", "SELECT 2"), excluded_fragments=("MODEL", "TEST ()"), ), diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py index eff258753e..0c5be7341e 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native.py @@ -36,9 +36,24 @@ expected_codes=(), ), LintNativeTestCase( - description="scenario without description does not fault", + description="scenario without description faults", contents='SCENARIO (tags ["x"]);\nSELECT 1\n', - expected_codes=(), + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="function without description faults", + contents="FUNCTION (\n arguments (amount INTEGER),\n returns BOOLEAN,\n);\namount > 1\n", + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="named hook without description faults", + contents="HOOK ();\nSELECT 1\n", + expected_codes=("description-present",), + ), + LintNativeTestCase( + description="header-only check flags a model_schema model for project resolution", + contents="MODEL (model_schema order_columns);\nSELECT 1 AS order_id\n", + expected_codes=("description-present",), ), LintNativeTestCase( description="long single-line scenario description faults by formatted length", diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index ebd928eea2..bfc918855c 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -37,6 +37,16 @@ class FormatProjectTestCase: expected_formatted_count: int = 0 +@dataclass(frozen=True) +class FormatDescriptionResolutionTestCase: + """Header description faults agree with the descriptions compile resolves.""" + + description: str + project_toml: str + files: dict[str, str] + expected_description_faults: tuple[tuple[str, str], ...] + + @dataclass(frozen=True) class FormatNewlineTestCase: """Test case for preserving newline conventions while formatting.""" diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index b81b9a9119..e85fc81211 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -514,12 +514,12 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select model_dir: Path = tmp_path / "models" model_dir.mkdir() model_dir.joinpath("stg_orders.sql").write_text( - "MODEL (columns (order_id (type INTEGER), status (type VARCHAR)));\n" + 'MODEL (description "Test model stg_orders.", columns (order_id (type INTEGER), status (type VARCHAR)));\n' "SELECT 1 AS order_id, 'open' AS status\n", encoding="utf-8", ) model_dir.joinpath("orders.sql").write_text( - "MODEL (columns (order_id (type INTEGER), status (type VARCHAR)));\n" + 'MODEL (description "Test model orders.", columns (order_id (type INTEGER), status (type VARCHAR)));\n' 'SELECT order_id, status FROM __ref("stg_orders")\n', encoding="utf-8", ) diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index b58d65fe4e..cf24ff4d79 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -12,6 +12,7 @@ from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from tests.unit.src.sqlbuild.lint._test_types import ( + FormatDescriptionResolutionTestCase, FormatNewlineTestCase, FormatProjectTestCase, LintBehaviorTestCase, @@ -90,10 +91,10 @@ def test_given_synthetic_project_when_linting_then_results_match_expected( expected_formatted_count=1, ), FormatProjectTestCase( - description="missing description remains a non-failing format warning", + description="missing description is a fault and the file is still formatted", files={"models/no_description.sql": NO_DESCRIPTION_MODEL}, expected_written_fragments={}, - expected_fault_codes=(), + expected_fault_codes=("description-present",), expected_formatted_count=1, ), FormatProjectTestCase( @@ -149,6 +150,64 @@ def test_given_synthetic_project_when_formatting_then_results_match_expected( assert fragment in written +@pytest.mark.parametrize( + "test_case", + [ + FormatDescriptionResolutionTestCase( + description="path default describes only the models under its path", + project_toml=PROJECT_TOML + '\n[path_defaults.marts]\ndescription = "Mart model"\n', + files={ + "models/marts/orders.sql": "MODEL ();\nSELECT 1 AS x\n", + "models/staging/stg_orders.sql": "MODEL ();\nSELECT 1 AS x\n", + }, + expected_description_faults=(("stg_orders.sql", "description-present"),), + ), + FormatDescriptionResolutionTestCase( + description="description resolution never imports project Python", + project_toml=PROJECT_TOML + '\n[path_defaults.marts]\ndescription = "Mart model"\n', + files={ + "providers/orders_api.py": 'raise RuntimeError("project Python was imported")\n', + "python/tasks/refresh.py": 'raise RuntimeError("project Python was imported")\n', + "models/marts/orders.sql": "MODEL ();\nSELECT 1 AS x\n", + "models/staging/stg_orders.sql": "MODEL ();\nSELECT 1 AS x\n", + }, + expected_description_faults=(("stg_orders.sql", "description-present"),), + ), + FormatDescriptionResolutionTestCase( + description="model schema description satisfies the header only when present", + project_toml=PROJECT_TOML, + files={ + "schemas/orders.sql": ( + 'SCHEMA (name order_shape, description "One row per order", ' + "columns (id (type INTEGER)));\n" + "SCHEMA (name bare_shape, columns (id (type INTEGER)));\n" + ), + "models/orders.sql": "MODEL (model_schema order_shape);\nSELECT 1 AS id\n", + "models/returns.sql": "MODEL (model_schema bare_shape);\nSELECT 1 AS id\n", + }, + expected_description_faults=(("returns.sql", "description-present"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_inherited_descriptions_when_formatting_then_faults_match_compile_resolution( + test_case: FormatDescriptionResolutionTestCase, tmp_path: Path +) -> None: + _ = (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") + relative_path: str + contents: str + for relative_path, contents in test_case.files.items(): + target: Path = tmp_path / relative_path + _ = target.parent.mkdir(parents=True, exist_ok=True) + _ = target.write_text(contents, encoding="utf-8") + + result: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig(), write=False) + + assert tuple(sorted((fault.file_path.name, fault.code) for fault in result.faults)) == ( + test_case.expected_description_faults + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index c1093a4554..a930e53846 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -191,7 +191,7 @@ def test_given_provider_class_imported_through_alias_when_calling_then_guides_us provider_file: Path = tmp_path / "providers" / "marker.py" provider_file.parent.mkdir(parents=True) provider_file.write_text( - "from sqlbuild.providers import Provider\n\nclass MarkerProvider(Provider):\n pass\n", + "from sqlbuild.providers import Provider\n\nclass MarkerProvider(Provider):\n '''Test provider MarkerProvider.'''\n pass\n", encoding="utf-8", ) old_path: list[str] = list(sys.path) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 7f6519a7dc..c31b0956bb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -60,7 +60,6 @@ "SQBRTEST102", "SQBRTEST103", "SQBRTEST104", - "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", "SQBRTEST203", @@ -79,7 +78,6 @@ "SQBRTEST102", "SQBRTEST103", "SQBRTEST104", - "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", "SQBRTEST203", diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index 6530d6355b..bfce6b065f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -33,7 +33,7 @@ rule_case=RuleCase( description="wrong configured domain faults", source=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "WITH final AS (SELECT 1 AS id)\n" "SELECT id FROM final\n" ), @@ -65,7 +65,7 @@ def test_given_rule_case_when_evaluating_then_runs_real_pipeline( rule_case=RuleCase( description="wrong configured domain faults", source=( - "MODEL (materialized table);\n\n" + "MODEL (description 'Test model.', materialized table);\n\n" "WITH final AS (SELECT 1 AS id)\n" "SELECT id FROM final\n" ), @@ -129,7 +129,7 @@ def test_given_same_custom_rule_when_using_public_and_native_paths_then_faults_h description="module-level state leaking between evaluations fails the harness", rule_case=RuleCase( description="first evaluation reports nothing", - source="MODEL (materialized table);\n\nSELECT 1 AS id\n", + source="MODEL (description 'Test model.', materialized table);\n\nSELECT 1 AS id\n", path="models/mart/commerce__mart__orders.sql", expected_finding_count=0, ), diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 8f1ac856b1..9024e609d5 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -57,7 +57,7 @@ def test_given_sql_analysis_state_when_running_rules_then_type_proof_rules_follo models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (\n" + "MODEL (description 'Test model orders.',\n" " contract enforced,\n" " columns (order_id (type INTEGER), status (type VARCHAR)),\n" ");\n" diff --git a/website/examples/hero-shop/models/finance/daily_revenue.sql b/website/examples/hero-shop/models/finance/daily_revenue.sql index 21cbbb30b2..7270016cb3 100644 --- a/website/examples/hero-shop/models/finance/daily_revenue.sql +++ b/website/examples/hero-shop/models/finance/daily_revenue.sql @@ -1,4 +1,5 @@ MODEL ( + description "Daily revenue from successful payments", materialized incremental, incremental_strategy delete_insert, cursor revenue_date, diff --git a/website/examples/tidy-shop/models/daily_revenue.sql b/website/examples/tidy-shop/models/daily_revenue.sql index 6440f96b40..9ce8506dab 100644 --- a/website/examples/tidy-shop/models/daily_revenue.sql +++ b/website/examples/tidy-shop/models/daily_revenue.sql @@ -1,4 +1,5 @@ MODEL ( + description "Daily revenue from successful payments", materialized table, ); diff --git a/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql b/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql index 7f1bc18a47..a2633f291e 100644 --- a/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql +++ b/website/examples/waffle-shop/functions/sql/table_fn__customer_orders.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Orders placed by one customer with their line totals", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, diff --git a/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql b/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql index 98ef857e40..a722c20280 100644 --- a/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql +++ b/website/examples/waffle-shop/functions/sql/udf__is_completed_order.sql @@ -1,4 +1,5 @@ FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); diff --git a/website/examples/waffle-shop/sources/raw.yml b/website/examples/waffle-shop/sources/raw.yml index 5bdbae9c55..69654f35a3 100644 --- a/website/examples/waffle-shop/sources/raw.yml +++ b/website/examples/waffle-shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw__customers + description: Raw customer records loaded by the Python loader managed: true write_strategy: table columns: @@ -15,6 +16,7 @@ sources: type: TIMESTAMP - name: raw__orders + description: Raw order records loaded by the Python loader managed: true write_strategy: delete_insert cursor_column: ordered_at @@ -33,6 +35,7 @@ sources: type: VARCHAR - name: raw__payments + description: Payment records for each order, defined inline expression: | SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index bafe69f9e4..b9417cff64 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -39,6 +39,14 @@ Configure the maximum physical line width in `sqlbuild_project.toml` (the defaul line_width = 100 ``` +`MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`; +`sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it +before compile does (see [Descriptions](/docs/concepts/models/configuration#descriptions)). A model +described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles. +Format does not render templates, so a templated description that renders empty passes format and +is reported by `sqb compile`. +Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault. + Long model and scenario descriptions are reflowed deterministically. Ordinary authored line breaks are normalized as spaces, while blank lines preserve paragraph boundaries. diff --git a/website/src/content/docs/docs/cli/scope.mdx b/website/src/content/docs/docs/cli/scope.mdx index 584ba79b4a..95f626b6e4 100644 --- a/website/src/content/docs/docs/cli/scope.mdx +++ b/website/src/content/docs/docs/cli/scope.mdx @@ -132,7 +132,7 @@ def formatted_order_total(expression: str) -> str: ```sql -- models/commerce/orders.sql -MODEL(); +MODEL(description "Orders"); SELECT @formatted_order_total("subtotal") AS total ``` diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 93ff76ab90..9b2cad7b76 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -48,6 +48,7 @@ Attach threshold and sample policy where the audit is used: ```sql MODEL ( + description "Valid order rate", audits [ valid_order_rate ( condition "order_id IS NOT NULL", @@ -112,6 +113,7 @@ Attach them in the `MODEL()` header like any generic audit: ```sql MODEL ( + description "One row per order", materialized view, tags [staging], columns ( @@ -194,6 +196,7 @@ that is not visible from there is a compile error. See ```sql MODEL ( + description "One row per order", materialized table, audits [ expression_is_true ( @@ -302,6 +305,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw__orders + description: Orders columns: - name: id audits: @@ -326,6 +330,7 @@ Seeds use the same YAML audit syntax as sources, at table and column level: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -379,6 +384,7 @@ Audits on incremental models can run at different lifecycle phases: ```sql MODEL ( + description "One row per order", materialized incremental, ... columns ( diff --git a/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx b/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx index 32f9ff8f05..74e89a373e 100644 --- a/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx +++ b/website/src/content/docs/docs/concepts/dbt-compatibility/adding-sqlbuild-models.mdx @@ -28,6 +28,7 @@ SQLBuild models reference dbt model outputs with `__dbt_ref("package", "model")` ```sql MODEL ( + description "Finance orders built on the dbt fact table", tags [finance], columns (order_id (audits [not_null])), ); @@ -40,7 +41,7 @@ This resolves to the qualified warehouse table name from the dbt manifest (e.g. SQLBuild models can also reference other SQLBuild models with `__ref()` as usual: ```sql -MODEL (tags [marts]); +MODEL (description "Mart view of downstream orders", tags [marts]); SELECT order_id FROM __ref("downstream_orders") ``` diff --git a/website/src/content/docs/docs/concepts/diff.mdx b/website/src/content/docs/docs/concepts/diff.mdx index 3e435a4246..6805e5e852 100644 --- a/website/src/content/docs/docs/concepts/diff.mdx +++ b/website/src/content/docs/docs/concepts/diff.mdx @@ -81,6 +81,7 @@ inherited sample and request exhaustive comparison with `row_diff_sample_rows 0` ```sql MODEL ( + description "One row per order", materialized table, unique_key [order_id, line_id], row_diff_sample_rows 0, @@ -145,6 +146,7 @@ Numeric columns can have tolerance rules to avoid false positives from floating- ```sql MODEL ( + description "Example model", materialized incremental, ... row_diff_tolerances ( @@ -169,6 +171,7 @@ Columns that are expected to differ between the two sides (like timestamps or co ```sql MODEL ( + description "One row per order", materialized incremental, ... row_diff_exclude_columns [latest_order_status], diff --git a/website/src/content/docs/docs/concepts/enums/model-contracts.mdx b/website/src/content/docs/docs/concepts/enums/model-contracts.mdx index e9a1a579d4..635a51819a 100644 --- a/website/src/content/docs/docs/concepts/enums/model-contracts.mdx +++ b/website/src/content/docs/docs/concepts/enums/model-contracts.mdx @@ -17,6 +17,7 @@ Use the enum name as the column type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/website/src/content/docs/docs/concepts/functions.mdx b/website/src/content/docs/docs/concepts/functions.mdx index 56f9f86665..07df454f94 100644 --- a/website/src/content/docs/docs/concepts/functions.mdx +++ b/website/src/content/docs/docs/concepts/functions.mdx @@ -11,11 +11,12 @@ Scalar UDFs return a single value per row. They can be written in SQL or Python. ### SQL UDFs -Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line: +Place SQL function files under `functions/sql/`. Each file has a `FUNCTION()` header declaring arguments and return type, followed by a SQL expression body. The header accepts `description`, `arguments`, `returns`, `database`, `schema`, and `tags`; any other key fails compilation with `D002`, naming the key and its file and line. `description` is required: a function without one fails compilation with `P010` (see [Descriptions](/docs/concepts/models/configuration#descriptions)): ```sql -- functions/sql/udf__is_completed_order.sql FUNCTION ( + description "Whether an order status counts as completed", arguments (order_status STRING), returns BOOLEAN, ); @@ -37,9 +38,12 @@ from sqlbuild.functions import udf runtime_version="3.11", ) def main(order_status: str | None) -> bool: + """Whether an order status counts as completed.""" return order_status == "completed" ``` +The function docstring is its description; `@udf(description="...")` overrides it. One of them is required, or compilation fails with `P010`. The docstring is deployed with the function body, so editing it redeploys the function; a `description=` value does not. + Python UDFs are deployed as warehouse-native functions (Snowflake Python UDFs, BigQuery remote functions, etc.). The `@udf` decorator is used for static discovery only - SQLBuild parses the AST without importing your code. ### Using scalar UDFs @@ -62,6 +66,7 @@ Table functions return multiple rows and columns. They are written in SQL under ```sql -- functions/sql/table_fn__customer_orders.sql FUNCTION ( + description "Customer orders", arguments (p_customer_id INTEGER), returns table ( order_id INTEGER, @@ -152,6 +157,7 @@ The `@udf` decorator accepts these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| +| `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | | `returns` | Yes | SQL return type string | | `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 8858022557..f98ac33f7b 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -17,6 +17,7 @@ Inserts new rows without modifying existing data. Optionally uses a cursor (read ```sql MODEL ( + description "One row per customer", materialized incremental, incremental_strategy append, cursor created_at, @@ -39,6 +40,7 @@ Deletes rows in the cursor range, then inserts the new delta. Requires either `c ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key [order_id], @@ -58,6 +60,7 @@ Upserts rows using a unique key. Matched rows are updated; unmatched rows are in ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy merge, unique_key [customer_id], @@ -94,6 +97,7 @@ Use `full_refresh false` for models that must retain their normal cursor or micr ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, full_refresh false, @@ -133,6 +137,7 @@ When a model references multiple upstream inputs, `cursor_inputs` is required to ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -154,6 +159,7 @@ Cursor-based incremental models can read their effective interval with zero-argu ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -191,6 +197,7 @@ Lookback extends the start of the replay window backwards to re-process recent d ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, cursor event_date, @@ -216,6 +223,7 @@ Each batch has its own audit cycle: create delta, run delta audits, apply DML, a ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -254,6 +262,7 @@ microbatch_concurrency = true ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -273,6 +282,7 @@ Watermark microbatch models can declare what to do when their resolved range con ```sql MODEL ( + description "One row per event", materialized incremental, incremental_strategy delete_insert, incremental_mode microbatch, @@ -328,6 +338,7 @@ Alignment does two things: it floors the replay window edges to the coarsest gra ```sql -- Upstream: daily grain, 2d batches MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, cursor activity_day, @@ -342,6 +353,7 @@ MODEL ( -- Downstream: hourly grain, but aligns to day automatically MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -369,6 +381,7 @@ The bounded duration supports `d` (days), `h` (hours), `m` (minutes), and `s` (s ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 57ee084ffc..9eccc3bc64 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -37,6 +37,7 @@ Use `@macro_name(...)` in a model query: ```sql MODEL ( + description "One row per order", materialized table, tags [marts], ); @@ -187,6 +188,7 @@ def grant_target(target: str) -> str: ```sql MODEL ( + description "Example model", post_hooks [inline_sql('@grant_target(@@CTX:destination.qualified)')], ); diff --git a/website/src/content/docs/docs/concepts/model-private-values.mdx b/website/src/content/docs/docs/concepts/model-private-values.mdx index 886f2a1670..baa5733254 100644 --- a/website/src/content/docs/docs/concepts/model-private-values.mdx +++ b/website/src/content/docs/docs/concepts/model-private-values.mdx @@ -8,6 +8,7 @@ project-wide namespace. ```sql MODEL ( + description "One row per order", enums ( _state [OPEN, CLOSED], ), @@ -48,6 +49,7 @@ Use `constant(...)` when a private constant needs an exact type or rendering cho ```sql MODEL ( + description "Example model", constants ( _usd_rate constant( type decimal, diff --git a/website/src/content/docs/docs/concepts/models/column-migrations.mdx b/website/src/content/docs/docs/concepts/models/column-migrations.mdx index 4fb598e3a2..05408005d0 100644 --- a/website/src/content/docs/docs/concepts/models/column-migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/column-migrations.mdx @@ -15,6 +15,7 @@ Add `migrate_from` to the renamed column in the model header, naming the old col ```sql MODEL ( + description "One row per order", materialized incremental, incremental_strategy delete_insert, unique_key order_id, diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 010e0a66d8..248f29979a 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -22,7 +22,7 @@ which accepts any keys. |-------|-------------| | `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | | `tags` | Tags used by selectors | -| `description` | Human-readable model description | +| `description` | Required human-readable model description; see [Descriptions](#descriptions) | | `columns` | Model-local column declarations, inherited-column audit augmentation, or column renames with `migrate_from` (incremental and snapshot models; see [Column migrations](/docs/concepts/models/column-migrations)). On a table or view model that declares its own `migrate_from`, a column `migrate_from` only names the old column in the [compatibility view](/docs/concepts/models/migrations#old-names) | | `model_schema` | Reusable column schema name | | `audits` | Model-level audit instances | @@ -39,6 +39,43 @@ which accepts any keys. | `dynamic_columns` | Named pivot column families with `pivot_column`, `value_column`, `aggregate`, `type`, and `name_pattern` | | `audit_factories` | Audit factories that generate audit cases for this model; see [Audits](/docs/concepts/audits) | +## Descriptions + +Every named resource must have a non-empty description. A missing description is a `P010` compile +error, so `sqb plan` and `sqb build` stop too. There is no setting to turn the check off. One +compile reports every missing description, each at the resource's declaration with the exact text +to add: + +```text +error[P010]: model 'orders_summary' has no description + --> models/marts/orders_summary.sql:1:1 + = help: to describe model 'orders_summary', add this to the MODEL header in models/marts/orders_summary.sql: + MODEL ( + description "What one row of orders_summary represents", + ... + ); +``` + +| Resource | Where the description goes | +|----------|----------------------------| +| Model | `MODEL (description "...")`, or the `description` of its `model_schema` | +| Scenario | `SCENARIO (description "...")` | +| Seed | `description:` on its entry in `seeds/*.yml` | +| Source | `description:` on its entry in `sources/*.yml` (or on the dlt resource) | +| SQL function or table function | `FUNCTION (description "...")` | +| Python UDF | `@udf(description="...")` or the function docstring | +| Named SQL hook | `HOOK (description "...")` | +| Python hook, task, asset, check | `description="..."` on the decorator or the function docstring | +| Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | +| Provider | The provider class docstring | + +SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a +description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a +model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it +does). `sqb format --check` also fails when a `MODEL`, `SCENARIO`, `FUNCTION` or `HOOK` +header has no description (a `MODEL` header that uses `model_schema` is left to compile), and +`sqb format` still moves a leading comment into an empty model description. + ## SQL analysis SQL analysis has project, invocation, and model gates: diff --git a/website/src/content/docs/docs/concepts/models/contracts.mdx b/website/src/content/docs/docs/concepts/models/contracts.mdx index 9a738a7620..aefe17679d 100644 --- a/website/src/content/docs/docs/concepts/models/contracts.mdx +++ b/website/src/content/docs/docs/concepts/models/contracts.mdx @@ -22,6 +22,7 @@ Under explicit mode, a model must declare `contract enforced` to activate shape ```sql MODEL ( + description "One row per order", materialized table, contract enforced, columns ( @@ -96,6 +97,7 @@ A column may use a declared enum as a portable logical domain type: ```sql MODEL ( + description "Example model", contract enforced, columns ( fulfillment_method (type fulfillment_method), diff --git a/website/src/content/docs/docs/concepts/models/hooks.mdx b/website/src/content/docs/docs/concepts/models/hooks.mdx index 1f52be6b81..575a0e3710 100644 --- a/website/src/content/docs/docs/concepts/models/hooks.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks.mdx @@ -77,6 +77,7 @@ Models may mix SQL and Python entries in one ordered hook list: ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql("INSERT INTO audit.build_log VALUES ('starting')"), diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx index 872516e359..c92be18ff3 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx @@ -21,13 +21,14 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, channel="#data-builds"): + """Notify complete.""" ctx.log( f"Notify {channel}: " f"{ctx.model_name} completed during {ctx.phase}" ) ``` -By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A Python file may define multiple decorated hooks. +By default, the hook name is the function name. The decorator accepts optional `name`, `description`, and `reads` arguments. A hook needs a description: pass `description=` or give the function a docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)). A Python file may define multiple decorated hooks. ## Declared reads @@ -41,6 +42,7 @@ from sqlbuild.refs import model @hook(reads=model("dim_customers")) def refresh_lookup(ctx): + """Refresh lookup.""" customers = ctx.relation(model("dim_customers")) ctx.execute_sql(f"CREATE OR REPLACE TABLE customer_lookup AS SELECT * FROM {customers}") ``` @@ -78,6 +80,7 @@ Reference the hook by name and optionally pass keyword arguments: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ python( @@ -132,6 +135,7 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send( f"Model {ctx.model_name} built successfully" ) diff --git a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx index f134a538c5..87e1eb7ed9 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/sql.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/sql.mdx @@ -25,7 +25,7 @@ GRANT SELECT ON @relation TO @role The hook name is always the filename stem, so this resource is invoked as `sql("grant_access", ...)`. Nested directories organize files but do not namespace names: `hooks/sql/admin/grant_access.sql` is still named `grant_access`. -`HOOK()` accepts only an optional, non-empty `description`. It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. +`HOOK()` accepts only a required, non-empty `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)). It does not accept a `name`; rename the file to rename the hook. The content after the header must be non-empty and becomes the SQL payload for each invocation. Files beginning with `_` are skipped. All other `.sql` files under `hooks/sql/` are parsed as hook resources and must have a valid `HOOK()` header. @@ -37,6 +37,7 @@ Pass the hook name and its arguments from a model's `pre_hooks` or `post_hooks` ```sql MODEL ( + description "Orders", materialized table, post_hooks [ sql( @@ -74,6 +75,7 @@ VALUES (@'relation', @'role') ```sql MODEL ( + description "Record access", post_hooks [ sql( "record_access", @@ -96,6 +98,7 @@ Use `inline_sql("...")` for SQL that is specific to one model: ```sql MODEL ( + description "Example model", materialized table, post_hooks [ inline_sql("GRANT SELECT ON @@CTX:destination.qualified TO analyst_role"), @@ -143,7 +146,7 @@ Context components can be combined into qualified identifiers without whitespace wrapper: ```sql -HOOK (); +HOOK (description "Create the reconstruct_book helper function"); CREATE FUNCTION @@CTX:destination.database.@@CTX:destination.schema.reconstruct_book() RETURNS INTEGER @@ -158,6 +161,7 @@ SQL hooks reference other resources the same way model SQL does, with `__ref(".. ```sql MODEL ( + description "One row per order", materialized table, pre_hooks [ inline_sql('DELETE FROM @@CTX:destination.qualified WHERE order_id IN (SELECT order_id FROM __ref("cancelled_orders"))'), @@ -171,7 +175,7 @@ SELECT order_id FROM __ref("stg_orders") **`models/marts/_sqlbuild/_hooks/sql/record_counts.sql`** ```sql -HOOK (); +HOOK (description "Record counts"); @count_rows(__ref("stg_orders")) ``` diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 397857d106..22d0aae228 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -10,7 +10,7 @@ The `materialized` field selects how a model becomes a warehouse relation. Creates or replaces a database view on each build: ```sql -MODEL (materialized view); +MODEL (description "One row per order", materialized view); SELECT id AS order_id, customer_id, status FROM __source("raw__orders") @@ -23,7 +23,7 @@ View audits run after the view has been replaced. A failing audit marks the buil By default, creates a staging table, applies supported type and contract enforcement, runs blocking audits, and only then promotes it to the destination. A pre-promotion failure leaves the previous destination unchanged. ```sql -MODEL (materialized table); +MODEL (description "One row per order", materialized table); SELECT customer_id, COUNT(*) AS total_orders FROM __ref("stg_orders") @@ -38,6 +38,7 @@ On a normal incremental run, applies append, delete/insert, or merge DML from a ```sql MODEL ( + description "Example model", materialized incremental, incremental_strategy delete_insert, cursor activity_hour, @@ -57,6 +58,7 @@ Maintains historical row versions with SCD Type 2 semantics: ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -75,6 +77,7 @@ A project-local Python materialization can manage specialized persistence with a ```sql MODEL ( + description "One row per order", materialized partition_tracked, placeholders ( partition_start "'2026-04-01'", diff --git a/website/src/content/docs/docs/concepts/models/migrations.mdx b/website/src/content/docs/docs/concepts/models/migrations.mdx index b142788471..4f2d9af630 100644 --- a/website/src/content/docs/docs/concepts/models/migrations.mdx +++ b/website/src/content/docs/docs/concepts/models/migrations.mdx @@ -15,6 +15,7 @@ To declare a migration by hand, add `migrate_from` to the renamed model's header ```sql MODEL ( + description "One row per order", materialized incremental, migrate_from stg_orders, cursor order_date, diff --git a/website/src/content/docs/docs/concepts/models/type-enforcement.mdx b/website/src/content/docs/docs/concepts/models/type-enforcement.mdx index 7bcbeea1f0..65fc5c323a 100644 --- a/website/src/content/docs/docs/concepts/models/type-enforcement.mdx +++ b/website/src/content/docs/docs/concepts/models/type-enforcement.mdx @@ -11,6 +11,7 @@ For a SQL model, declaring a type on any inline or reusable-schema column enable ```sql MODEL ( + description "One row per order", materialized table, columns ( order_id (type INTEGER), diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx index 31f0c61a10..50fcbb7b43 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx @@ -16,6 +16,7 @@ from sqlbuild.assets import asset, AssetContext @asset def orders_export(ctx: AssetContext): + """Orders export.""" path = write_orders_csv() return ctx.result(metadata={"path": path, "rows": 1200}, materialized=True) ``` @@ -29,6 +30,7 @@ Assets record whether they actually produced an artifact via `materialized`: ```python @asset def orders_export(ctx: AssetContext): + """Orders export.""" if nothing_changed(): return ctx.result(metadata={"status": "unchanged"}, materialized=False) return ctx.result(metadata={"path": export()}, materialized=True) @@ -50,6 +52,7 @@ from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_dashboard(ctx): + """Orders dashboard.""" result = ctx.result_of(export_orders) return ctx.result(metadata={"rows": result.payload["rows"]}, materialized=True) ``` @@ -63,6 +66,7 @@ from sqlbuild.refs import model @asset(depends_on=model("fact_orders")) def orders_extract(ctx): + """Orders extract.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT count(*) FROM {relation}").fetchone()[0] return ctx.result(metadata={"rows": rows}, materialized=True) @@ -84,6 +88,7 @@ Assets can declare a schema for catalog and lineage purposes. This does not enfo }, ) def orders_export(ctx): + """Orders export.""" ... ``` @@ -103,6 +108,7 @@ from sqlbuild.retries import RetryPolicy @asset(retry=RetryPolicy(max_attempts=3, retry_on=(IOError,))) def export(ctx): + """Export the order summary.""" if not ready(): return ctx.skip("upstream not ready", mode="soft") # or SkipMode.SOFT return ctx.result(metadata={"path": do_export()}, materialized=True) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index 668acfd5f3..c4d5355e9e 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -19,6 +19,7 @@ from python.tasks.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): + """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: return ctx.fail("no orders exported") @@ -36,6 +37,7 @@ from sqlbuild.refs import model @check(depends_on=model("fact_orders")) def orders_present(ctx): + """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] return ctx.pass_() if count else ctx.fail("no orders") @@ -50,6 +52,7 @@ Return a result through the context helpers, or a bool shorthand: ```python @check(depends_on=orders_asset) def rows_present(ctx): + """Rows present.""" return ctx.result_of(orders_asset).payload["rows"] > 0 # True -> pass, False -> fail ``` @@ -70,6 +73,7 @@ Returning `None` is not allowed - checks must be explicit. ```python @check(depends_on=export_orders, severity="warn") def orders_freshness(ctx): + """Orders freshness.""" if stale(): return ctx.fail("export is stale") # recorded as a warning, does not fail the build return ctx.pass_() diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index c2af78862c..6994e8fbee 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -25,6 +25,7 @@ def export_assets(): for table in TABLES: @asset(name=f"export_{table}", tags=("export",)) def export(ctx, table=table): + """Export one table to object storage.""" return ctx.result(metadata={"table": table}, materialized=True) nodes.append(export) return nodes @@ -56,14 +57,17 @@ from sqlbuild.checks import check def orders_pipeline(): @loader(name="raw_orders") def load(ctx): + """Load orders from the storefront API.""" return fetch_orders() @asset(name="orders_export", depends_on=load) def export(ctx): + """Export the order summary.""" return ctx.result(materialized=True) @check(depends_on=export) def orders_export_check(ctx): + """Check that the orders export is ready.""" return ctx.pass_("export ready") return [load, export, orders_export_check] diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 2c1e44c548..8f8da8d450 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -24,6 +24,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_customers(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customers from the CRM export.""" return [ {"id": 1, "name": "Leslie Knope", "email": "leslie@pawnee.gov"}, {"id": 2, "name": "Ron Swanson", "email": "ron@pawnee.gov"}, @@ -39,6 +40,7 @@ Declare a managed source in `sources/*.yml`. A managed source is bound to the lo ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -69,6 +71,7 @@ Full replace. The target is dropped and recreated from the loader output on ever ```yaml sources: - name: raw_countries + description: Countries managed: true write_strategy: table columns: @@ -85,6 +88,7 @@ Insert all returned rows into the target. No deduplication. ```yaml sources: - name: raw_webhook_events + description: Webhook events managed: true write_strategy: append columns: @@ -101,6 +105,7 @@ Delete rows in the cursor range, then insert replacements. Requires `cursor_colu ```yaml sources: - name: raw_order_events + description: Order events managed: true write_strategy: delete_insert cursor_column: event_at @@ -118,6 +123,7 @@ The loader receives `ctx.current_cursor_value` with the current `MAX(cursor_colu ```python @loader def raw_order_events(ctx: LoaderContext) -> list[dict[str, object]]: + """Order events.""" if ctx.current_cursor_value is None: return fetch_all_events() return fetch_events_since(ctx.current_cursor_value) @@ -130,6 +136,7 @@ Upsert based on `unique_key`. Requires both `unique_key` and `cursor_column`. ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: merge unique_key: customer_id @@ -152,6 +159,7 @@ If a loader returns `None`, SQLBuild skips its row-writing pipeline. The loader ```python @loader def raw_status(ctx: LoaderContext) -> None: + """Load the order status lookup.""" ctx.execute_sql(f"DROP TABLE IF EXISTS {ctx.destination}") ctx.execute_sql( f"CREATE TABLE {ctx.destination} AS " @@ -164,6 +172,7 @@ The source is still declared as managed, just without a `write_strategy`: ```yaml sources: - name: raw_status + description: Order status lookup managed: true columns: - name: status_id @@ -234,6 +243,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" return [ {"account_id": 1, "account_name": "Pawnee Parks"}, {"account_id": 2, "account_name": "Eagleton"}, @@ -241,6 +251,7 @@ def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: @loader(depends_on=[raw_accounts]) def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" accounts = ctx.loader(raw_accounts) rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") return [ @@ -258,6 +269,7 @@ Intermediate loaders (those referenced only via `depends_on`, with no managed so ```python @loader(destination="staging.shared_accounts") def raw_accounts(ctx: LoaderContext): + """Load customer accounts.""" ... ``` @@ -274,6 +286,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | `unique_key` | Column(s) used as the merge key (string or list of strings) | | `columns` | Column specifications with name, type, nullable, and description | | `contract` | `enforced` or `none` | +| `description` | What the loaded rows are; defaults to the function docstring. Required unless the source YAML entry has a `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)) | ## Auto-load during builds diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 9ac5a63176..5087778955 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -78,7 +78,7 @@ Every decorator accepts the same organizational metadata: | `depends_on` | A single function, tuple, or list of upstream nodes (and, where allowed, `model()`/`source()`/`seed()` references) | | `tags` | Labels for selection, filtering, and catalog grouping | | `group` | A display/catalog grouping string | -| `description` | Human-readable docs (defaults to the function docstring) | +| `description` | Required human-readable docs; defaults to the function docstring ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `meta` | Freeform JSON metadata for catalogs and integrations | `@task` and `@asset` also accept a `retry` policy. `@asset` additionally accepts `columns` and `column_lineage`. Node kind is inferred from the decorator - you never pass `kind=`. @@ -119,6 +119,7 @@ Tasks and assets return through `ctx.result(...)`: ```python @task def export_orders(ctx): + """Export orders.""" return ctx.result(payload={"rows": 120}, metadata={"rows": 120}) ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 1019f76824..2eb9b35d0c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -14,6 +14,7 @@ Create a Python file under `providers/` in your project. A provider is a class t from sqlbuild.providers import Provider class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str endpoint: str = "https://api.example.com" @@ -24,6 +25,8 @@ class WarehouseClient(Provider): self.session.close() ``` +The class docstring is the provider's description and is required; a provider without one fails compilation with `P010` ([Descriptions](/docs/concepts/models/configuration#descriptions)). + `Provider` extends [pydantic-settings `BaseSettings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/), so provider fields are validated and can be populated from environment variables automatically. ## Provider name @@ -37,6 +40,7 @@ Override the name explicitly with `provider_name`: ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" provider_name = "warehouse" api_key: str ``` @@ -52,6 +56,7 @@ from sqlbuild.tasks import task @task def export_orders(ctx, warehouse_client): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -65,6 +70,7 @@ from providers.warehouse_client import WarehouseClient @task def export_orders(ctx, warehouse_client: WarehouseClient): + """Export orders.""" warehouse_client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -87,11 +93,13 @@ from sqlbuild.hooks import hook @hook def notify_complete(ctx, slack_notifier): + """Notify complete.""" slack_notifier.send(f"Model {ctx.model_name} built successfully") ``` ```sql MODEL ( + description "One row per order", materialized table, post_hooks [python("notify_complete")], ); @@ -102,6 +110,7 @@ Providers are also available on the `HookContext` via `ctx.providers`: ```python @hook def notify_complete(ctx): + """Notify complete.""" notifier = ctx.providers.slack_notifier notifier.send(f"Model {ctx.model_name} built successfully") ``` @@ -113,6 +122,7 @@ All Python node contexts (`TaskContext`, `AssetContext`, `CheckContext`, `Loader ```python @task def export_orders(ctx): + """Export orders.""" client = ctx.providers.warehouse_client client.session.upload(ctx.query("SELECT * FROM orders")) ``` @@ -129,6 +139,7 @@ Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command ```python class WarehouseClient(Provider): + """Client for the reporting warehouse.""" api_key: str def setup(self, ctx): @@ -148,6 +159,7 @@ Because `Provider` extends `pydantic-settings BaseSettings`, fields without defa ```python class SlackNotifier(Provider): + """Posts build notifications to a chat channel.""" slack_token: str # reads SLACK_TOKEN from environment channel: str = "#builds" # has a default, environment variable is optional ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx index 7c31978f8e..e0101e9095 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/sql-references.mdx @@ -18,16 +18,19 @@ from sqlbuild.tasks import task @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" ... @task(depends_on=source("raw_orders")) def inspect_raw(ctx): + """Inspect raw.""" ... @task(depends_on=seed("country_codes")) def export_countries(ctx): + """Export countries.""" ... ``` @@ -47,6 +50,7 @@ Always resolve a reference to its concrete relation with `ctx.relation(...)` ins ```python @task(depends_on=model("fact_orders")) def export_orders(ctx): + """Export orders.""" relation = ctx.relation(model("fact_orders")) rows = ctx.query(f"SELECT * FROM {relation}").fetchall() return ctx.result(metadata={"rows": len(rows)}) diff --git a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx index 836879bdfc..f7f4c4860a 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx @@ -16,6 +16,7 @@ from sqlbuild.tasks import task, TaskContext @task def export_orders(ctx: TaskContext): + """Export orders.""" rows = fetch_orders() return ctx.result(payload={"rows": len(rows)}, metadata={"rows": len(rows)}) ``` @@ -29,11 +30,13 @@ Tasks declare dependencies with `depends_on`, accepting a single function, a tup ```python @task def fetch_orders(ctx: TaskContext): + """Fetch orders.""" return ctx.result(payload=download_orders()) @task(depends_on=fetch_orders) def summarize_orders(ctx: TaskContext): + """Summarize orders.""" result = ctx.result_of(fetch_orders) return ctx.result(metadata={"count": len(result.payload)}) ``` @@ -47,6 +50,7 @@ Tasks may depend on other tasks, assets, and loaders. They may **not** depend on ```python @task def build_export(ctx: TaskContext): + """Build export.""" return ctx.result( payload={"path": "/exports/orders.csv"}, metadata={"rows": 1200}, @@ -64,6 +68,7 @@ Return `ctx.skip(...)` to skip a task: ```python @task def export_if_present(ctx: TaskContext): + """Export if present.""" if not new_files_available(): return ctx.skip("no new files") return ctx.result(payload=do_export()) @@ -80,6 +85,7 @@ from sqlbuild.tasks import task, SkipMode @task def optional_step(ctx): + """Optional step.""" return ctx.skip("nothing to do", mode=SkipMode.HARD) # or mode="hard" ``` @@ -94,6 +100,7 @@ from sqlbuild.tasks import task @task(retry=RetryPolicy(max_attempts=3, retry_on=(ConnectionError,))) def call_api(ctx): + """Call api.""" return ctx.result(payload=fetch_from_flaky_api()) ``` diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 44e414a286..c38af35958 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -32,6 +32,11 @@ XSQBR001 → XSQBR Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching semantics and takes precedence over `select`. +A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired +when descriptions became the compile error `P010` +([Descriptions](/docs/concepts/models/configuration#descriptions)); remove it from `select` and +`ignore`. + Use the catalogue to inspect available codes and metadata: ```bash diff --git a/website/src/content/docs/docs/concepts/scenarios.mdx b/website/src/content/docs/docs/concepts/scenarios.mdx index b7dd4bbfeb..cd6934f6eb 100644 --- a/website/src/content/docs/docs/concepts/scenarios.mdx +++ b/website/src/content/docs/docs/concepts/scenarios.mdx @@ -108,7 +108,7 @@ The header is metadata only: | Field | Description | |-------|-------------| -| `description` | Optional description string | +| `description` | Required description of the behaviour the scenario proves ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `tags` | Optional list of string tags | ## How scenarios execute diff --git a/website/src/content/docs/docs/concepts/seeds.mdx b/website/src/content/docs/docs/concepts/seeds.mdx index 05d864b6d1..cfbe20b4b9 100644 --- a/website/src/content/docs/docs/concepts/seeds.mdx +++ b/website/src/content/docs/docs/concepts/seeds.mdx @@ -43,7 +43,7 @@ Seed YAML accepts only these keys; any other key is a compile error: | Seed | `name`, `description`, `database`, `schema`, `meta`, `tags`, `csv_settings`, `columns`, `audits` | | Column | `name`, `type`, `nullable`, `description`, `meta`, `audits` | -Every seed must have a YAML declaration with at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). +Every seed must have a YAML declaration with a non-empty `description` ([Descriptions](/docs/concepts/models/configuration#descriptions)) and at least one typed column. CSV filenames must be unique across the entire `seeds/` directory (including subdirectories). The seed name in the YAML must match the CSV filename (without the `.csv` extension). ## Referencing seeds @@ -77,6 +77,7 @@ Attach audits to a seed or its columns with the same syntax as source audits: ```yaml seeds: - name: waffle_types + description: Waffle types columns: - name: waffle_type_id type: INTEGER @@ -99,6 +100,7 @@ Seeds inherit the project's default database and schema. You can override these ```yaml seeds: - name: waffle_types + description: Waffle types database: analytics schema: lookups columns: @@ -115,6 +117,7 @@ For non-standard CSV formats, configure parsing behavior with `csv_settings`: ```yaml seeds: - name: european_prices + description: European prices csv_settings: delimiter: ";" encoding: utf-8 diff --git a/website/src/content/docs/docs/concepts/snapshots.mdx b/website/src/content/docs/docs/concepts/snapshots.mdx index 119e0cf7c8..2e1ec9dd75 100644 --- a/website/src/content/docs/docs/concepts/snapshots.mdx +++ b/website/src/content/docs/docs/concepts/snapshots.mdx @@ -24,6 +24,7 @@ Use when your source has a reliable column recording when the entity changed. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -47,6 +48,7 @@ Use when the source does not have a reliable update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -75,6 +77,7 @@ Use for daily full exports or periodic snapshots without a business update times ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy check, @@ -98,6 +101,7 @@ Use for historical observations that include a business update timestamp. ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -123,6 +127,7 @@ Use for CDC tables, audit logs, or historical backfills where rows are individua ```sql MODEL ( + description "One row per order", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -228,6 +233,7 @@ The model `snapshot_full_refresh` field can only make the policy **stricter** th ```sql MODEL ( + description "Example model", materialized snapshot, ... snapshot_full_refresh deny, @@ -250,6 +256,7 @@ Snapshot models support the same audit system as other materializations. Audits ```sql MODEL ( + description "One row per customer", materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, @@ -324,6 +331,7 @@ JOIN customer_snapshot c ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [user_id, role_id], snapshot_strategy check, @@ -341,6 +349,7 @@ FROM __source("user_role_daily") ```sql MODEL ( + description "One row per product", materialized snapshot, unique_key [product_id], snapshot_strategy timestamp, diff --git a/website/src/content/docs/docs/concepts/sources.mdx b/website/src/content/docs/docs/concepts/sources.mdx index 598f8e95ef..1c34ec0595 100644 --- a/website/src/content/docs/docs/concepts/sources.mdx +++ b/website/src/content/docs/docs/concepts/sources.mdx @@ -12,6 +12,7 @@ Point at an existing table or view in your warehouse: ```yaml sources: - name: raw_events + description: Events database: analytics schema: raw table: events @@ -26,6 +27,7 @@ Define source data inline as a SQL expression. No external tables or setup scrip ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT * FROM (VALUES (1, 'Leslie', 'Knope', 'leslie@pawnee.gov', TIMESTAMP '2026-01-15 09:00:00'), @@ -46,6 +48,7 @@ Sources support the same audit system as models. Audits attached to sources run ```yaml sources: - name: raw_orders + description: Orders columns: - name: id audits: @@ -69,6 +72,7 @@ Type enforcement is implicit for sources. If any column declares a `type`, SQLBu ```yaml sources: - name: raw__customers + description: Customers expression: | SELECT 1 AS id, 'Leslie' AS first_name, 'Knope' AS last_name columns: @@ -91,6 +95,7 @@ Sources can be loaded by Python functions instead of pointing at existing tables ```yaml sources: - name: raw_customers + description: Customers managed: true write_strategy: table columns: @@ -122,6 +127,7 @@ Configure freshness per source with a `freshness:` block: ```yaml sources: - name: raw_events + description: Events schema: raw table: events freshness: @@ -232,7 +238,7 @@ Use [`sqb freshness`](/docs/cli/freshness) to observe source freshness on demand | `cursor_column` | Column for incremental cursor tracking (required for `delete_insert` and `merge`) | | `unique_key` | Merge key column(s) (required for `merge`) | | `freshness` | Source freshness observation config (see [Source freshness](#source-freshness)) | -| `description` | Human-readable description | +| `description` | Required human-readable description ([Descriptions](/docs/concepts/models/configuration#descriptions)) | | `type_enforcement` | Override implicit type enforcement (`true`/`false`). Defaults to `true` when any column declares a type. | | `contract` | `enforced` or `none`. When enforced, downstream models validate configured column references against source columns. | | `columns` | Column declarations with `name`, `type`, `nullable`, `description`, `meta`, and `audits` | diff --git a/website/src/content/docs/docs/integrations/dagster-reference.mdx b/website/src/content/docs/docs/integrations/dagster-reference.mdx index 7e88ef5a43..6f845dcf7a 100644 --- a/website/src/content/docs/docs/integrations/dagster-reference.mdx +++ b/website/src/content/docs/docs/integrations/dagster-reference.mdx @@ -207,6 +207,7 @@ A source can provide a stable Dagster-native key without replacing the translato ```yaml sources: - name: raw__orders + description: Orders meta: dagster: asset_key: [warehouse, raw, orders] diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 2f8e03e7fa..c1e4b8cc4e 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -45,6 +45,7 @@ dlt_sources: type: header_link resources: - name: raw_github_issues + description: Issues from the GitHub API write_disposition: append endpoint: path: "repos/${github_owner}/${github_repo}/issues" @@ -65,6 +66,7 @@ dlt_sources: credentials: "${postgres_connection_string}" resources: - name: raw_customers + description: Customers from the CRM database table: customers write_disposition: merge primary_key: id @@ -82,6 +84,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + description: Events from the event export files reader: csv write_disposition: append ``` @@ -93,6 +96,7 @@ dlt_sources: | Key | Description | |-----|-------------| | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | +| `description` | What the loaded rows are. Required ([Descriptions](/docs/concepts/models/configuration#descriptions)). | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | @@ -144,6 +148,7 @@ from sqlbuild.executor.load.models import LoaderContext @loader def raw_github_issues(ctx: LoaderContext): + """Github issues.""" source = rest_api_source({ "client": { "base_url": "https://api.github.com/", @@ -178,6 +183,7 @@ Bind it to a source with `managed: true` in `sources/*.yml`: # sources/github.yml sources: - name: raw_github_issues + description: Github issues managed: true table: issues columns: diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index d218c3cb3e..47f0b7e9d0 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -38,6 +38,7 @@ Replicate a table from PostgreSQL into your local DuckDB project: # sources/raw.yml sources: - name: raw_orders + description: Orders table: orders ingestr: source_uri: "postgresql://user:pass@host:5432/mydb" @@ -53,6 +54,7 @@ Load Stripe charges with incremental merge on a primary key: ```yaml sources: - name: raw_stripe_charges + description: Stripe charges table: charges ingestr: source_uri: "stripe://${stripe_api_key}" @@ -95,6 +97,7 @@ All string fields in the `ingestr` block support SQLBuild project variable subst ```yaml sources: - name: raw_orders + description: Orders ingestr: source_uri: "postgresql://${pg_user}:${pg_password}@${pg_host}:5432/${pg_database}" source_table: "public.orders"