From 04ba3ba9f5a5bddfc594373cdfaf470722435108 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:49:50 +0100 Subject: [PATCH 01/17] feat!: add typed SQL macros and require @py./@sql. macro calls Add SQL macros: logic-free SQL fragments in macros/sql/ with a MACRO header and typed arguments, in three forms decided by the body (an expression, a WITH list of CTEs, or a SELECT list of columns). Calls are written @sql.name(...), expand in place in the declaration pass, and are checked on their own and again in every caller. Parameters are written @name in bodies; types follow schema declaration types, with CONST for compile-time scalars, LIST/SET, ENUM, RELATION and COLUMN OF arguments. Runtime parameters in literal-only positions are rejected. Formatting, rules and column lineage see through calls, and macro-mode unit tests and __macro__ mocks work for SQL macros. BREAKING CHANGE: Python macros now live in macros/python/ (and the python/ folder of scoped macro roots) and are called as @py.name(...). A macro file directly under a macros/ folder fails with D013, and a bare @name(...) call fails with P014; move files into macros/python/ and replace @name( with @py.name( in project SQL. --- benchmarks/dbt-compare/write_dbt_dense.py | 4 +- .../sql_tests/authored_macro_calls.rs | 10 + .../compiler/_helpers/sql_tests/extraction.rs | 9 + .../src/compiler/tests/helpers.rs | 2 +- .../tests/test_sql_test_extraction_rules.rs | 10 +- .../tests/test_sql_test_helper_references.rs | 2 +- crates/sqlbuild-analysis/src/lib.rs | 1 + .../tests/test_function_names.rs | 2 +- .../src/sql_macros/_helpers/arguments.rs | 619 ++ .../src/sql_macros/_helpers/body.rs | 307 + .../src/sql_macros/_helpers/calls.rs | 118 + .../src/sql_macros/_helpers/definition.rs | 384 ++ .../src/sql_macros/_helpers/expansion.rs | 470 ++ .../src/sql_macros/_helpers/header.rs | 420 ++ .../src/sql_macros/_helpers/lexing.rs | 228 + .../sql_macros/_helpers/literal_positions.rs | 242 + .../src/sql_macros/_helpers/mod.rs | 10 + .../src/sql_macros/_helpers/parameters.rs | 125 + .../src/sql_macros/_helpers/placement.rs | 363 + .../src/sql_macros/constants.rs | 77 + .../src/sql_macros/errors.rs | 91 + .../src/sql_macros/main/bare_macro_calls.rs | 47 + .../main/declaration_visible_from_macro.rs | 42 + .../sql_macros/main/expand_sql_macro_call.rs | 22 + .../src/sql_macros/main/mod.rs | 8 + .../src/sql_macros/main/parse_sql_macro.rs | 12 + .../main/render_inline_collection.rs | 30 + .../src/sql_macros/main/sql_macro_calls.rs | 10 + .../src/sql_macros/main/sql_macro_cycle.rs | 50 + .../main/with_list_sql_macro_calls.rs | 35 + .../sqlbuild-analysis/src/sql_macros/mod.rs | 11 + .../src/sql_macros/models.rs | 241 + .../sqlbuild-analysis/src/sql_macros/tests.rs | 8 + .../src/sql_macros/tests/helpers.rs | 118 + .../src/sql_macros/tests/test_definitions.rs | 295 + .../src/sql_macros/tests/test_expansion.rs | 419 ++ .../src/sql_macros/tests/test_types.rs | 48 + .../sqlbuild-analysis/src/sql_macros/types.rs | 60 + .../test_sql_test_classification_cases.rs | 26 +- .../src/audits/_helpers/parameters.rs | 11 +- .../tests/test_render_parameterized_sql.rs | 4 +- .../src/digest/tests/test_content_digest.rs | 6 +- .../src/format_command/constants.rs | 3 +- .../src/init_command/constants.rs | 7 +- crates/sqlbuild-discovery/src/constants.rs | 6 +- .../tests/test_discovery_session.rs | 5 +- .../declarations/_helpers/declaration_tree.rs | 66 +- .../tests/test_declaration_layout.rs | 37 +- .../model_files/tests/test_header_facts.rs | 4 +- .../config_presence/_helpers/macro_calls.rs | 14 + .../tests/test_config_presence.rs | 12 +- .../tests/test_build_model_config.rs | 2 +- .../compile_driver/assembly_checks.rs | 10 +- .../_helpers/compile_records/constants.rs | 1 + .../_helpers/input_loops/model_loop.rs | 9 + .../_helpers/input_loops/model_loop_hooks.rs | 12 + .../_helpers/input_loops/model_loop_host.rs | 8 +- .../_helpers/input_loops/model_loop_macros.rs | 2 +- .../_helpers/input_loops/source_loop.rs | 5 + .../_helpers/input_loops/sql_test_loop.rs | 31 +- .../_helpers/record_behaviours/error_inits.rs | 4 +- .../bindings/_helpers/records/construct.rs | 21 + .../src/bindings/_helpers/records/fields.rs | 5 + .../bindings/_helpers/render/authored_sql.rs | 9 +- .../_helpers/render/declaration_expansion.rs | 31 + .../_helpers/render/declaration_values.rs | 7 + .../_helpers/render/macro_evaluation.rs | 14 +- .../_helpers/render/macro_expansion.rs | 60 +- .../render/macro_generated_references.rs | 4 +- .../src/bindings/_helpers/render/mod.rs | 4 + .../bindings/_helpers/render/registration.rs | 3 +- .../_helpers/render/sql_macro_host.rs | 532 ++ .../_helpers/render/sql_macro_inventory.rs | 76 + .../_helpers/render/sql_macro_mocks.rs | 75 + .../_helpers/render/sql_macro_tests.rs | 59 + .../sqltext/declaration_references.rs | 59 +- .../_helpers/scanning/interpolation.rs | 13 +- .../src/refactoring/tests/helpers.rs | 2 +- .../src/refactoring/tests/test_sql_sites.rs | 8 +- .../src/macro_arguments/_helpers/failures.rs | 2 +- .../tests/test_parse_macro_arguments.rs | 14 +- .../src/macro_calls/_helpers/scanner.rs | 99 +- .../sqlbuild-render/src/macro_calls/errors.rs | 6 + .../src/macro_calls/main/macro_call_names.rs | 6 +- .../sqlbuild-render/src/macro_calls/models.rs | 2 +- .../src/macro_calls/tests/helpers.rs | 8 +- .../macro_calls/tests/test_macro_call_memo.rs | 16 +- .../tests/test_macro_call_names.rs | 24 +- .../tests/test_scan_macro_call_sites.rs | 120 +- .../src/dsl_headers/_helpers/rules.rs | 5 +- .../src/dsl_headers/constants.rs | 6 +- .../src/engine/tests/test_evaluation.rs | 2 +- .../src/engine/tests/test_sql_scanning.rs | 48 +- .../src/lint_text/_helpers/format_bodies.rs | 51 +- .../src/lint_text/_helpers/sites.rs | 44 +- .../sqlbuild-rules/src/lint_text/constants.rs | 2 + .../main/neutralize_audit_parameters.rs | 31 +- .../tests/test_fixture_formatting.rs | 2 +- .../src/lint_text/tests/test_lint_text.rs | 28 +- .../src/sql_lint/_helpers/formatter_syntax.rs | 54 +- .../_helpers/declaration_directory.rs | 20 + .../placement/main/placement_diagnostics.rs | 5 +- .../src/placement/tests/helpers.rs | 14 +- .../src/placement/tests/test_placement.rs | 10 +- .../relationship_names/_helpers/cte_scan.rs | 33 + .../tests/test_expected_model_names.rs | 4 +- .../tests/test_relationship_facts.rs | 6 +- .../src/scope_report/_helpers/declarations.rs | 17 +- fensu.toml | 1 + .../_helpers/dense_project.py | 13 +- .../corpus/attachment_failure_cases.py | 4 +- .../_helpers/corpus/config_failure_cases.py | 2 +- .../corpus/discovery_failure_cases.py | 6 +- .../_helpers/corpus/failure_cases.py | 112 +- .../_helpers/corpus/render_failure_cases.py | 214 +- .../_helpers/corpus/scope_failure_cases.py | 18 +- .../_helpers/coverage/render.py | 4 +- .../classes/analysis_features.py | 4 +- .../classes/discovery_features.py | 16 +- .../classes/project_builder.py | 19 +- .../classes/render_features.py | 44 +- scripts/compiler_differential/constants.py | 23 +- scripts/rules_benchmark/_helpers/workflow.py | 4 +- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/scope.md | 20 +- .../docs/concepts/declaration-scopes.md | 2 +- .../concepts/declaration-scopes/explorer.md | 36 +- .../concepts/declaration-scopes/visibility.md | 10 +- .../references/docs/concepts/interpolation.md | 4 +- .../references/docs/concepts/macros.md | 59 +- .../macros/composition-and-context.md | 14 +- .../docs/concepts/macros/sql-macros.md | 264 + .../references/docs/concepts/models.md | 2 +- .../docs/concepts/models/hooks/sql.md | 4 +- .../docs/concepts/project-configuration.md | 2 +- .../references/docs/concepts/testing.md | 67 +- src/sqlbuild/_native.pyi | 5 + .../compile/_helpers/render/user_macros.py | 129 +- .../macros/{ => python}/currency.py.txt | 0 .../_macros/{ => python}/currency.py.txt | 0 .../_macros/{ => python}/datetime.py.txt | 0 .../models/marts/daily_activity_rollup.sql | 4 +- .../models/marts/daily_revenue.sql | 2 +- .../waffle_shop/models/marts/fact_orders.sql | 2 +- .../hourly_activity_with_daily_context.sql | 2 +- .../models/marts/hourly_order_activity.sql | 4 +- .../unit/test_line_total_cents_macro.sql | 2 +- .../macros/{ => python}/currency.py | 0 .../models/marts/_macros/{ => python}/time.py | 0 .../models/marts/daily_activity_rollup.sql | 4 +- .../waffle_shop/models/marts/fact_orders.sql | 2 +- .../hourly_activity_with_daily_context.sql | 2 +- .../models/marts/hourly_order_activity.sql | 4 +- .../unit/test_line_total_cents_macro.sql | 2 +- .../cli/commands/main/build/helpers.py | 4 +- .../main/build/test_explicit_references.py | 12 +- .../main/build/test_sql_hook_reads.py | 4 +- .../cli/commands/main/clickhouse/helpers.py | 2 +- .../test_native_attachment_errors.py | 6 +- .../cli/commands/main/compile/helpers.py | 73 +- .../main/compile/sql_macros/__init__.py | 0 .../main/compile/sql_macros/_test_types.py | 57 + .../main/compile/sql_macros/helpers.py | 167 + .../sql_macros/test_sql_macro_build.py | 347 + .../sql_macros/test_sql_macro_errors.py | 647 ++ .../sql_macros/test_sql_macro_unit_tests.py | 155 + .../test_native_sql_test_errors.py | 6 +- .../test_compile_cache_invalidation.py | 4 +- .../compile/test_compile_incremental_edits.py | 8 +- .../compile/test_compile_macro_call_store.py | 22 +- .../main/compile/test_compile_reuse.py | 8 +- .../main/compile/test_native_config_errors.py | 10 +- .../compile/test_native_declaration_errors.py | 8 +- .../main/compile/test_native_macro_errors.py | 48 +- .../compile/test_native_reference_errors.py | 11 +- .../cli/commands/main/refactor/helpers.py | 14 +- .../main/refactor/test_move_declarations.py | 8 +- .../main/refactor/test_rename_columns.py | 2 +- .../cli/commands/main/rules/helpers.py | 4 +- .../rules/test_rules_cache_invalidation.py | 2 +- .../main/rules/test_sql_rule_paths.py | 4 +- .../cli/commands/main/scope/helpers.py | 9 +- .../cli/commands/main/scope/test_scope.py | 8 +- .../cli/commands/main/sqlite/helpers.py | 2 +- .../cli/commands/main/sqlserver/helpers.py | 2 +- .../cli/commands/main/test/helpers.py | 20 +- .../main/test/test_commented_fixtures.py | 14 +- .../main/test/test_direct_logic_extraction.py | 12 +- .../main/test/test_optional_test_ceremony.py | 2 +- .../main/test/test_set_operation_expected.py | 6 +- ..._expansion_nested_argument_scan_edges.json | 30 +- .../macro_expansion_seeded_fragments.json | 5894 ++++++++--------- ...r_a_model_with_a_non_ascii_name_after.json | 4 +- ...lper_whose_nested_cte_shadows_another.json | 2 +- ...helper_with_an_invalid_reference_call.json | 2 +- ...he_default_context_rounds_and_flushes.json | 2 +- ...meterized_and_mock_reads_helper_tests.json | 12 +- ..._folded_reads_and_located_diagnostics.json | 2 +- .../src/sqlbuild/cli/commands/main/helpers.py | 4 +- .../cli/commands/main/test_compile.py | 14 +- .../main/test_compile_model_headers.py | 2 +- .../commands/main/test_contract_command.py | 4 +- .../main/test_dollar_quoted_literals.py | 4 +- .../sqlbuild/cli/commands/main/test_format.py | 10 +- .../cli/commands/main/test_format_fix.py | 14 +- .../commands/main/test_format_preservation.py | 26 +- .../commands/main/test_repeated_json_parse.py | 4 +- .../sqlbuild/cli/commands/main/test_rules.py | 22 +- .../cli/entry/test_native_launcher.py | 8 +- .../sqlbuild/compiler/attachments/helpers.py | 44 +- .../src/sqlbuild/compiler/compile/helpers.py | 77 +- .../compile/test_native_macro_bridge.py | 52 +- .../compiled_project/expected/parity.json | 6 +- .../compiled_project/expected/sql_hooks.json | 2 +- .../expected/waffle_shop.json | 24 +- .../compiler/compiled_project/helpers.py | 14 +- .../test_native_compiled_project.py | 16 +- .../compiled_project/test_native_sources.py | 2 +- .../test_native_declaration_files.py | 12 +- .../sqlbuild/compiler/model_config/helpers.py | 4 +- .../model_config/test_config_values.py | 2 +- .../model_config/test_native_model_config.py | 2 +- .../test_macro_declaration_context.py | 76 +- .../pipeline/test_macro_load_count.py | 14 +- .../sqlbuild/compiler/pipeline/test_main.py | 4 +- .../src/sqlbuild/compiler/scopes/helpers.py | 33 +- .../scopes/test_native_scope_parity.py | 8 +- .../compiler/sql_test_glue/helpers.py | 8 +- .../host_payload/test_host_payload_facts.py | 4 +- .../_helpers/coverage/test_discovery.py | 2 +- .../_helpers/coverage/test_render.py | 14 +- .../commands/_helpers/lineage/test_cache.py | 6 +- .../commands/_helpers/scope/test_output.py | 2 +- .../cli/commands/main/init/test_run.py | 6 +- .../main/playground/test_playground.py | 6 +- .../compiler/compile/_helpers/helpers.py | 16 +- .../compile/_helpers/test_assembly.py | 6 +- .../_helpers/test_declaration_scopes.py | 16 +- .../compile/_helpers/test_expansion_spans.py | 14 +- .../compile/_helpers/test_hook_reads.py | 6 +- .../compiler/compile/_helpers/test_macros.py | 172 +- .../_helpers/test_parameterized_sql_tests.py | 8 +- .../compile/_helpers/test_sql_vars.py | 4 +- .../_helpers/test_typed_macro_references.py | 56 +- .../sqlbuild/compiler/compile/test_main.py | 58 +- .../discovery/_helpers/_test_helpers.py | 2 +- .../compiler/discovery/_helpers/helpers.py | 6 + .../_helpers/test_filesystem_macros.py | 10 +- .../test_filesystem_scoped_declarations.py | 59 +- .../discovery/_helpers/test_sql_hooks.py | 4 +- .../discovery/_helpers/test_sql_tests.py | 2 +- .../sqlbuild/compiler/discovery/test_main.py | 4 +- .../compiler/macro_bridge/_helpers/helpers.py | 4 +- .../_helpers/test_store_environment.py | 2 +- .../macro_bridge/_helpers/test_store_keys.py | 8 +- .../macro_bridge/classes/test_macro_bridge.py | 2 +- .../sqlbuild/compiler/manifest/test_main.py | 8 +- .../_helpers/sql_test_assembly/helpers.py | 4 +- .../sql_test_assembly/test_plan_test.py | 4 +- .../src/sqlbuild/compiler/scopes/helpers.py | 6 +- .../scopes/test_builder_and_visibility.py | 2 +- .../compiler/scopes/test_offline_cache.py | 64 +- .../compiler/scopes/test_placement.py | 37 +- .../compiler/scopes/test_relocation.py | 4 +- .../sqlbuild/lint/_helpers/test_expansion.py | 2 +- .../lint/_helpers/test_sqlbuild_tokens.py | 44 +- .../unit/src/sqlbuild/lint/test_cli_format.py | 8 +- .../src/sqlbuild/lint/test_expanded_lint.py | 16 +- .../macros/{ => python}/currency.py | 0 .../_macros/{ => python}/currency.py | 0 .../_macros/{ => python}/datetime.py | 0 .../models/marts/daily_order_rollup.sql | 4 +- .../models/marts/daily_revenue.sql | 2 +- .../waffle-shop/models/marts/fact_orders.sql | 2 +- .../hourly_activity_with_daily_context.sql | 2 +- .../models/marts/hourly_order_activity.sql | 4 +- .../unit/test_line_total_cents_macro.sql | 2 +- website/src/content/docs/docs/cli/scope.mdx | 20 +- .../docs/docs/concepts/declaration-scopes.mdx | 2 +- .../concepts/declaration-scopes/explorer.mdx | 36 +- .../declaration-scopes/visibility.mdx | 10 +- .../docs/docs/concepts/interpolation.mdx | 4 +- .../src/content/docs/docs/concepts/macros.mdx | 61 +- .../macros/composition-and-context.mdx | 14 +- .../docs/docs/concepts/macros/sql-macros.mdx | 251 + .../src/content/docs/docs/concepts/models.mdx | 2 +- .../docs/docs/concepts/models/hooks/sql.mdx | 4 +- .../docs/concepts/project-configuration.mdx | 2 +- .../content/docs/docs/concepts/testing.mdx | 67 +- website/src/sidebar.json | 5 +- 290 files changed, 13098 insertions(+), 4291 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/constants.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/errors.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/declaration_visible_from_macro.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_cycle.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/mod.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/models.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_inventory.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_mocks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_tests.rs create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md rename src/sqlbuild/playground_templates/waffle_shop/macros/{ => python}/currency.py.txt (100%) rename src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/{ => python}/currency.py.txt (100%) rename src/sqlbuild/playground_templates/waffle_shop/models/marts/_sqlbuild/_macros/{ => python}/datetime.py.txt (100%) rename tests/e2e/fixtures/waffle_shop/macros/{ => python}/currency.py (100%) rename tests/e2e/fixtures/waffle_shop/models/marts/_macros/{ => python}/time.py (100%) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_unit_tests.py rename website/examples/waffle-shop/macros/{ => python}/currency.py (100%) rename website/examples/waffle-shop/models/marts/_sqlbuild/_macros/{ => python}/currency.py (100%) rename website/examples/waffle-shop/models/marts/_sqlbuild/_macros/{ => python}/datetime.py (100%) create mode 100644 website/src/content/docs/docs/concepts/macros/sql-macros.mdx diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py index 6fa7cbb0d7..9c5d33ba0e 100644 --- a/benchmarks/dbt-compare/write_dbt_dense.py +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -43,7 +43,7 @@ COLUMN_RE = re.compile( r'^\s+(\w+) \(type (\w+), nullable (true|false), description "([^"]*)"(, audits \[not_null\])?\)' ) -MACRO_CALL_RE = re.compile(r'@(macro_\d{3}_\d{5})\("([^"]*)"\)') +MACRO_CALL_RE = re.compile(r'@py\.(macro_\d{3}_\d{5})\("([^"]*)"\)') CAST_RE = re.compile(r"CAST\(([^ ]+) AS (\w+)\) AS (\w+)") @@ -201,7 +201,7 @@ def write_dbt(sqb: Path, out: Path, n: int) -> Path: base_re = re.compile( r'def (base_offset_\d{3})\(expression: str\) -> str:\n return f"\(\{expression\} \+ 1\)"' ) - for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/*/*.py")): + for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/python/*/*.py")): text = macro.read_text() name, ret = macro_def_re.search(text).groups() offset = offset_re.search(text)[1] diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs index cfa25123d3..16f0a1fe9b 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs @@ -47,6 +47,16 @@ fn call_open(sql: &str, at: usize) -> Option { return None; } let mut cursor = at + 1 + first.len_utf8(); + for prefix in ["py.", "sql."] { + if sql[at + 1..].starts_with(prefix) { + cursor = at + 1 + prefix.len(); + let first = char_at(sql, cursor)?; + if !(first == '_' || first.is_alphabetic()) { + return None; + } + cursor += first.len_utf8(); + } + } while let Some(character) = char_at(sql, cursor) { if !is_identifier_continue(character) { break; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index f98885a529..243271e3df 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -1226,6 +1226,15 @@ fn macro_calls(sql: &str, syntax: &LexicalSyntax) -> Result, String> if byte_at(sql, index) == Some(b'@') && let Some((name, end)) = read_identifier(sql, index + 1, syntax) { + let (name, end) = + if matches!(name.as_str(), "py" | "sql") && byte_at(sql, end) == Some(b'.') { + match read_identifier(sql, end + 1, syntax) { + Some(called) => called, + None => (name, end), + } + } else { + (name, end) + }; let open = space_end(sql, end); if byte_at(sql, open) == Some(b'(') && !DECLARATION_CALLS.contains(&name.as_str()) { if !calls.contains(&name) { diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index 305d2fa2e7..ef30aaadb8 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -56,7 +56,7 @@ pub(crate) fn empty_model_fixture_marker_preserves_direct_mode_validation() -> b pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { let cases = [ ( - r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#, + r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @py.status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#, "SQL test 'tests/status.sql' must alias every non-trivial __macro_expected__ projection", ), ( diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs index b1125b3969..3936828345 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs @@ -42,7 +42,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' CTE name 'my$helper' must be an unquoted identifier of ASCII letters, digits and underscores", NAME_HELP, @@ -284,7 +284,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call udf; call reusable logic only in __macro_actual__", None, @@ -316,7 +316,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH h AS (SELECT @tidy(a) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH h AS (SELECT @py.tidy(a) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call macros; call macros only in __macro_actual__", None, @@ -328,7 +328,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "udf", raw: true, syntax: "generic", - sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @var('x') AS a)", + sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @py.var('x') AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'udf' CTE __udf_expected__ must not call macros", None, @@ -340,7 +340,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "udf", raw: true, syntax: "generic", - sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@m(1)) AS b)", + sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@py.m(1)) AS b)", expected_outcome: accepted_outcome(false), }, ExtractionRuleTestCase { diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs index e2d0a816ce..b6e1ffeaff 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -580,7 +580,7 @@ fn given_compiled_sql_test_inputs_when_assembling_then_facts_match_python() { model( "order_dollars", &[], - Some("SELECT '@cents(x)', @dollars (amount)"), + Some("SELECT '@py.cents(x)', @py.dollars (amount)"), &[], ), ] diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index af65f6af67..376fd5733b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -43,6 +43,7 @@ pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod snowflake_cost; +pub mod sql_macros; pub mod sql_test_attachment; pub mod sql_tokens; pub mod target_paths; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs index 6761b1fbd3..11698e22b1 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs @@ -128,7 +128,7 @@ fn given_supported_or_non_builtin_call_when_scanning_then_reports_nothing() { description: "methods, qualified functions, sentinels and macros are not checked", dialect: DialectType::Snowflake, sql: "SELECT tools.starts_with(a, 'x'), analytics.helpers.lcase(a), \ - __sqlbuild_udf_starts_with(a, 'x'), @starts_with(a), payload:lcase(a) \ + __sqlbuild_udf_starts_with(a, 'x'), @py.starts_with(a), payload:lcase(a) \ FROM __ref(\"products\")", expected_calls: &[], }, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs new file mode 100644 index 0000000000..c696813797 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs @@ -0,0 +1,619 @@ +//! A call's argument list: split, bound to parameters and checked against their types. + +use crate::sql_macros::_helpers::header::{split_top_level, unquoted}; +use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::constants::{ + ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, +}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, BoundArgument, CallArgument, ParameterKind, SqlMacroDefinition, + SqlMacroParameter, TypeFamily, +}; +use crate::type_system::main::normalize_type::normalize_type; + +const LITERAL_WORDS: [&str; 3] = ["TRUE", "FALSE", "NULL"]; +const TYPED_LITERAL_WORDS: [&str; 5] = ["DATE", "TIMESTAMP", "TIME", "INTERVAL", "TIMESTAMPTZ"]; + +/// Each top-level argument of `arguments` with its `name :=` prefix, if any. +pub(crate) fn split_arguments(arguments: &str) -> Result, SqlMacroError> { + let scanned: Vec = tokens(arguments).map_err(|_| { + SqlMacroError::new( + ARGUMENT_CODE, + "macro call arguments have an unclosed quote or comment", + ) + })?; + let mut found: Vec = Vec::new(); + for part in split_top_level(arguments, &scanned) { + let named: bool = part.len() >= NAMED_ARGUMENT_MIN_TOKENS + && part[0].kind == TokenKind::Word + && part[1].is(arguments, ":="); + let (name, value): (Option, &[Token]) = if named { + (Some(part[0].text(arguments).to_owned()), &part[2..]) + } else { + (None, part) + }; + let (Some(first), Some(last)) = (value.first(), value.last()) else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + "a named macro argument needs a value after :=", + )); + }; + found.push(CallArgument { + name, + text: arguments[first.start..last.end].to_owned(), + offset: first.start, + }); + } + Ok(found) +} + +/// Each parameter's argument: positional first, then named, then its default. +pub(crate) fn bind_arguments( + definition: &SqlMacroDefinition, + arguments: &[CallArgument], +) -> Result, SqlMacroError> { + let name: &str = &definition.name; + let mut bound: Vec> = vec![None; definition.parameters.len()]; + let mut seen_named: bool = false; + for (position, argument) in arguments.iter().enumerate() { + let index: usize = match &argument.name { + Some(argument_name) => { + seen_named = true; + definition + .parameters + .iter() + .position(|parameter| parameter.name.eq_ignore_ascii_case(argument_name)) + .ok_or_else(|| { + SqlMacroError::new( + ARGUMENT_CODE, + format!("macro {name}() has no argument {argument_name}"), + ) + .helped(format!("its arguments are {}", parameter_names(definition))) + })? + } + None => { + if seen_named { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "positional argument after a named argument in a call to macro {name}()" + ), + )); + } + if position >= definition.parameters.len() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "macro {name}() takes {} argument{}; the call passes {}", + definition.parameters.len(), + if definition.parameters.len() == 1 { + "" + } else { + "s" + }, + arguments.len() + ), + )); + } + position + } + }; + if bound[index].is_some() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {} of macro {name}() is passed twice", + definition.parameters[index].name + ), + )); + } + bound[index] = Some(BoundArgument { + parameter: index, + text: argument.text.clone(), + from_default: false, + offset: Some(argument.offset), + }); + } + let mut complete: Vec = Vec::with_capacity(bound.len()); + for (index, argument) in bound.into_iter().enumerate() { + let parameter: &SqlMacroParameter = &definition.parameters[index]; + match (argument, ¶meter.default) { + (Some(argument), _) => complete.push(argument), + (None, Some(default)) => complete.push(BoundArgument { + parameter: index, + text: default.clone(), + from_default: true, + offset: None, + }), + (None, None) => { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "call to macro {name}() is missing argument {} {}", + parameter.name, + parameter.kind.label() + ), + )); + } + } + } + Ok(complete) +} + +fn parameter_names(definition: &SqlMacroDefinition) -> String { + if definition.parameters.is_empty() { + return "none".to_owned(); + } + definition + .parameters + .iter() + .map(|parameter| parameter.name.as_str()) + .collect::>() + .join(", ") +} + +/// How an argument is written, read from its tokens alone. +pub(crate) fn argument_shape(text: &str) -> ArgumentShape { + let Ok(scanned) = tokens(text) else { + return ArgumentShape::Expression; + }; + match scanned.as_slice() { + [] => ArgumentShape::Expression, + [single] => match single.kind { + TokenKind::Number | TokenKind::Text => ArgumentShape::Literal, + TokenKind::Word + if LITERAL_WORDS + .iter() + .any(|word| single.text(text).eq_ignore_ascii_case(word)) => + { + ArgumentShape::Literal + } + TokenKind::Word | TokenKind::Quoted => ArgumentShape::Identifier { + name: single.text(text).to_owned(), + }, + _ => ArgumentShape::Expression, + }, + [at, word] if at.is(text, "@") && word.kind == TokenKind::Word && word.start == at.end => { + ArgumentShape::ParameterReference { + name: word.text(text).to_owned(), + } + } + [sign, number] if sign.is(text, "-") && number.kind == TokenKind::Number => { + ArgumentShape::Literal + } + [word, value] + if word.kind == TokenKind::Word + && value.kind == TokenKind::Text + && TYPED_LITERAL_WORDS + .iter() + .any(|typed| word.text(text).eq_ignore_ascii_case(typed)) => + { + ArgumentShape::Literal + } + [at, keyword, open, name, close] + if at.is(text, "@") + && keyword.is_word(text, "const") + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") => + { + ArgumentShape::ConstantReference { + name: unquote(name.text(text)), + } + } + [at, keyword, open, name, close, dot, member] + if at.is(text, "@") + && keyword.is_word(text, "enum") + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") + && dot.is(text, ".") + && member.kind == TokenKind::Word => + { + ArgumentShape::EnumMember { + name: unquote(name.text(text)), + member: member.text(text).to_owned(), + } + } + [first, ..] + if (first.is(text, "[") || first.is(text, "{")) + && closing_token(text, &scanned, 0) == Some(scanned.len() - 1) => + { + ArgumentShape::InlineCollection + } + [function, open, name, close] + if function.kind == TokenKind::Word + && REFERENCE_FUNCTIONS.contains(&function.text(text)) + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") => + { + ArgumentShape::Reference { + call: text.trim().to_owned(), + } + } + _ => ArgumentShape::Expression, + } +} + +fn unquote(text: &str) -> String { + crate::sql_macros::_helpers::header::unquoted(text) +} + +/// The type family of a SQL literal argument. +fn literal_family(text: &str) -> TypeFamily { + let trimmed: &str = text.trim(); + let upper: String = trimmed.to_ascii_uppercase(); + if upper == NULL_WORD { + return TypeFamily::Null; + } + if BOOLEAN_WORDS.contains(&upper.as_str()) { + return TypeFamily::Boolean; + } + if trimmed.starts_with('\'') || trimmed.starts_with('$') { + return TypeFamily::Text; + } + if let Some(word) = upper.split_whitespace().next() + && TYPED_LITERAL_WORDS.contains(&word) + { + return match word { + "DATE" => TypeFamily::Date, + "TIME" => TypeFamily::Time, + "INTERVAL" => TypeFamily::Unknown, + _ => TypeFamily::Timestamp, + }; + } + let digits: &str = trimmed.trim_start_matches('-').trim(); + if digits.contains(['.', 'e', 'E']) { + TypeFamily::Decimal + } else { + TypeFamily::Integer + } +} + +/// The adapter whose type rules apply and the calling macro an `@name` argument forwards from. +#[derive(Clone, Copy)] +pub(crate) struct ShapeContext<'a> { + pub(crate) adapter: &'a str, + pub(crate) caller: Option<&'a SqlMacroDefinition>, +} + +/// Whether `shape` is something `parameter` accepts; the caller checks references it resolves. +pub(crate) fn check_shape( + definition: &SqlMacroDefinition, + parameter: &SqlMacroParameter, + (argument, shape): (&str, &ArgumentShape), + context: ShapeContext<'_>, +) -> Result<(), SqlMacroError> { + let macro_name: &str = &definition.name; + let name: &str = ¶meter.name; + let shown: &str = argument.trim(); + let expects = |accepts: &str| { + SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {}: {accepts}; got {shown}", + parameter.kind.label() + ), + ) + }; + match (¶meter.kind, shape) { + (_, ArgumentShape::ParameterReference { name: forwarded }) => { + let Some(outer) = context + .caller + .and_then(|caller| caller.parameter(forwarded).map(|found| (caller, found))) + else { + return Ok(()); + }; + if forwards(&outer.1.kind, ¶meter.kind) { + Ok(()) + } else { + Err(expects(&format!( + "a value of that kind; @{forwarded} is {} argument {} of macro {}()", + article(&outer.1.kind.label()), + outer.1.name, + outer.0.name + )) + .helped(match ¶meter.kind { + ParameterKind::Constant { .. } => format!( + "declare {} CONST in macro {}() so it is known at compile time", + outer.1.name, outer.0.name + ), + _ => "forward an argument of the same kind".to_owned(), + })) + } + } + (ParameterKind::Constant { data_type }, ArgumentShape::EnumMember { .. }) => { + if fits(TypeFamily::Text, type_family(data_type, context.adapter)) { + Ok(()) + } else { + Err(expects("a literal, a scalar constant or an enum value of that type")) + } + } + (ParameterKind::Expression { data_type }, ArgumentShape::Literal) + | (ParameterKind::Constant { data_type }, ArgumentShape::Literal) => { + let actual: TypeFamily = literal_family(argument); + if fits(actual, type_family(data_type, context.adapter)) { + Ok(()) + } else { + Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {data_type}; {shown} is {}", + family_label(actual) + ), + )) + } + } + (ParameterKind::Expression { .. }, ArgumentShape::InlineCollection) => { + Err(expects("a SQL expression")) + } + (ParameterKind::Expression { .. }, _) => Ok(()), + (ParameterKind::Constant { .. }, ArgumentShape::ConstantReference { .. }) => Ok(()), + (ParameterKind::Constant { .. }, _) => Err(expects("a literal, a scalar constant or an enum value") + .helped(format!("pass a literal, such as 2, or a constant, such as @const(\"{name}\"); a CONST argument is known at compile time, so a column or expression cannot be passed"))), + ( + ParameterKind::Collection { .. }, + ArgumentShape::InlineCollection | ArgumentShape::ConstantReference { .. }, + ) => Ok(()), + (ParameterKind::Collection { .. }, _) => { + Err(expects("an inline list or set, or a list or set constant") + .helped("pass a list, such as [\"GB\", \"FR\"], or a constant, such as @const(\"supported_countries\")")) + } + (ParameterKind::Enum { name: enum_name }, ArgumentShape::EnumMember { name: given, .. }) + if given == enum_name => + { + Ok(()) + } + (ParameterKind::Enum { name: enum_name }, _) => Err(expects(&format!( + "a member of enum {enum_name}, such as @enum(\"{enum_name}\").MEMBER" + ))), + ( + ParameterKind::Relation { .. }, + ArgumentShape::Identifier { .. } | ArgumentShape::Reference { .. }, + ) => Ok(()), + (ParameterKind::Relation { .. }, _) => Err(expects( + "a CTE name, a table alias, or __ref(), __source() or __seed()", + )), + (ParameterKind::ColumnOf { .. }, ArgumentShape::Identifier { .. }) => Ok(()), + (ParameterKind::ColumnOf { relation }, _) => { + Err(expects(&format!("a column name of {relation}"))) + } + } +} + +/// Whether an argument of kind `outer`, forwarded as `@name`, fits a parameter of kind `inner`. +fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { + match inner { + ParameterKind::Expression { .. } => matches!( + outer, + ParameterKind::Expression { .. } + | ParameterKind::Constant { .. } + | ParameterKind::Enum { .. } + | ParameterKind::ColumnOf { .. } + ), + ParameterKind::Constant { .. } => { + matches!( + outer, + ParameterKind::Constant { .. } | ParameterKind::Enum { .. } + ) + } + ParameterKind::Collection { .. } => matches!(outer, ParameterKind::Collection { .. }), + ParameterKind::Enum { name } => { + matches!(outer, ParameterKind::Enum { name: outer_name } if outer_name == name) + } + ParameterKind::Relation { .. } => matches!(outer, ParameterKind::Relation { .. }), + ParameterKind::ColumnOf { .. } => matches!(outer, ParameterKind::ColumnOf { .. }), + } +} + +fn article(label: &str) -> String { + let vowel: bool = label.starts_with(['A', 'E', 'I', 'O', 'U']); + format!("{} {label}", if vowel { "an" } else { "a" }) +} + +/// A type family as messages name it. +pub(crate) fn family_label(family: TypeFamily) -> &'static str { + match family { + TypeFamily::Integer => "an integer", + TypeFamily::Decimal => "a decimal number", + TypeFamily::Float => "a floating-point number", + TypeFamily::Text => "a string", + TypeFamily::Boolean => "a boolean", + TypeFamily::Date => "a date", + TypeFamily::Timestamp => "a timestamp", + TypeFamily::Time => "a time", + TypeFamily::Null => "NULL", + TypeFamily::Unknown => "of an unknown type", + } +} + +/// Whether the SQL text is one self-delimited term that needs no parentheses around it. +pub(crate) fn is_atomic(sql: &str) -> bool { + let Ok(scanned) = tokens(sql) else { + return false; + }; + let Some(first) = scanned.first() else { + return false; + }; + if matches!( + argument_shape(sql), + ArgumentShape::ParameterReference { .. } + ) { + return true; + } + if scanned.len() == 1 { + return matches!( + first.kind, + TokenKind::Word | TokenKind::Quoted | TokenKind::Number | TokenKind::Text + ); + } + if first.is(sql, "(") { + return closing_token(sql, &scanned, 0) == Some(scanned.len() - 1); + } + if first.is_word(sql, "CASE") { + let mut depth: isize = 0; + for (index, token) in scanned.iter().enumerate() { + if token.is_word(sql, "CASE") { + depth += 1; + } else if token.is_word(sql, "END") { + depth -= 1; + if depth == 0 { + return index == scanned.len() - 1; + } + } + } + return false; + } + let mut index: usize = 0; + while index < scanned.len() { + let token: &Token = &scanned[index]; + if !matches!(token.kind, TokenKind::Word | TokenKind::Quoted) { + return false; + } + match scanned.get(index + 1) { + None => return true, + Some(next) if next.is(sql, ".") => index += 2, + Some(next) if next.is(sql, "(") => { + return closing_token(sql, &scanned, index + 1) == Some(scanned.len() - 1); + } + Some(_) => return false, + } + } + false +} + +/// The family of a declared type, normalized as schema declaration types are for the adapter. +pub(crate) fn type_family(data_type: &str, adapter: &str) -> TypeFamily { + use crate::type_system::models::TypeFamily as Normalized; + let dialect: &str = match adapter { + "motherduck" => "duckdb", + "" => "generic", + other => other, + }; + let normalized = normalize_type(data_type, dialect) + .or_else(|_| normalize_type(data_type, "generic")) + .map(|normalization| normalization.normalized.family); + match normalized { + Ok(Normalized::Integer) => TypeFamily::Integer, + Ok(Normalized::Decimal) => TypeFamily::Decimal, + Ok(Normalized::Float) => TypeFamily::Float, + Ok(Normalized::String) => TypeFamily::Text, + Ok(Normalized::Boolean) => TypeFamily::Boolean, + Ok(Normalized::Date) => TypeFamily::Date, + Ok(Normalized::Timestamp | Normalized::Datetime) => TypeFamily::Timestamp, + Ok(Normalized::Other) | Err(_) => TypeFamily::Unknown, + } +} + +/// Whether a value of type `actual` may be passed where `expected` is declared. +pub(crate) fn fits(actual: TypeFamily, expected: TypeFamily) -> bool { + use TypeFamily::{ + Boolean, Date, Decimal, Float, Integer, Null, Text, Time, Timestamp, Unknown, + }; + matches!( + (actual, expected), + (Unknown | Null, _) + | (_, Unknown) + | (Integer, Integer | Decimal | Float) + | (Decimal | Float, Decimal | Float) + | (Date, Date | Timestamp) + | (Timestamp, Timestamp) + | (Time, Time) + | (Text, Text) + | (Boolean, Boolean) + ) +} + +/// The rendered literal and type family of each value of an inline collection's inner tokens. +pub(crate) fn collection_items( + text: &str, + inner: &[Token], + invalid: &dyn Fn(&str) -> SqlMacroError, +) -> Result, SqlMacroError> { + let mut items: Vec<(String, TypeFamily)> = Vec::new(); + let mut index: usize = 0; + while index < inner.len() { + let negative: bool = inner[index].is(text, "-"); + if negative { + index += 1; + } + let value: &Token = inner + .get(index) + .ok_or_else(|| invalid("ends with a sign"))?; + items.push( + collection_item(text, value, negative) + .ok_or_else(|| invalid("holds a value that is not a literal"))?, + ); + index += 1; + match inner.get(index) { + None => {} + Some(comma) if comma.is(text, ",") => index += 1, + Some(_) => return Err(invalid("needs commas between its values")), + } + } + Ok(items) +} + +/// One collection value as a SQL literal with its type family, if it is a literal. +fn collection_item(text: &str, value: &Token, negative: bool) -> Option<(String, TypeFamily)> { + match value.kind { + TokenKind::Quoted | TokenKind::Text => Some(( + format!("'{}'", unquoted(value.text(text)).replace('\'', "''")), + TypeFamily::Text, + )), + TokenKind::Number => { + let digits: &str = value.text(text); + let family: TypeFamily = if digits.contains(['.', 'e', 'E']) { + TypeFamily::Decimal + } else { + TypeFamily::Integer + }; + Some(( + format!("{}{digits}", if negative { "-" } else { "" }), + family, + )) + } + TokenKind::Word if !negative => match value.text(text).to_ascii_lowercase().as_str() { + "true" => Some(("TRUE".to_owned(), TypeFamily::Boolean)), + "false" => Some(("FALSE".to_owned(), TypeFamily::Boolean)), + "null" => Some(("NULL".to_owned(), TypeFamily::Null)), + _ => None, + }, + _ => None, + } +} + +/// Why a collection's values do not form one list or set, if they do not. +pub(crate) fn collection_problem(items: &[(String, TypeFamily)], set: bool) -> Option { + let typed: Vec = items + .iter() + .map(|(_, family)| *family) + .filter(|family| *family != TypeFamily::Null) + .collect(); + let Some(first) = typed.first() else { + return Some("needs at least one value that is not null".to_owned()); + }; + if typed + .iter() + .any(|family| family != first && !(is_numeric(*family) && is_numeric(*first))) + { + return Some("mixes values of different types".to_owned()); + } + if !set { + return None; + } + items.iter().enumerate().find_map(|(position, (value, _))| { + repeats(&items[..position], value) + .then(|| format!("repeats {value}; a set's values must be unique")) + }) +} + +fn is_numeric(family: TypeFamily) -> bool { + matches!(family, TypeFamily::Integer | TypeFamily::Decimal) +} + +fn repeats(earlier: &[(String, TypeFamily)], value: &str) -> bool { + earlier.iter().any(|(seen, _)| seen == value) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs new file mode 100644 index 0000000000..238aca8e35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs @@ -0,0 +1,307 @@ +//! A SQL macro body parsed as SQL, its parameter references located and replaced in place. + +use polyglot_sql::expressions::{Identifier, Select}; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; + +use crate::sql_macros::_helpers::arguments::is_atomic; +use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parameters::{ + ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, + with_placeholders, +}; +use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; +use crate::sql_macros::errors::LexError; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, BodyExpansion, ParameterKind, RenderedArgument, SqlMacroDefinition, + SqlMacroForm, SqlMacroParameter, +}; + +/// Dialects tried in turn to parse a body; the first that parses it locates its references. +const PARSE_DIALECTS: [DialectType; 8] = [ + DialectType::Generic, + DialectType::DuckDB, + DialectType::Snowflake, + DialectType::BigQuery, + DialectType::PostgreSQL, + DialectType::Databricks, + DialectType::ClickHouse, + DialectType::TSQL, +]; +const EXPRESSION_PREFIX: &str = "SELECT "; +const CTE_SUFFIX: &str = "\nSELECT 1"; + +/// The body parsed as SQL. +struct ParsedBody { + select: Box