diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 0869c385bb..2023792121 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -186,7 +186,7 @@ jobs: done unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \ - --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "crates/sqlbuild-rules-native/Cargo.toml")] | join("\n")')" + --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "Cargo.toml")] | join("\n")')" if [[ -n "${unexpected_files}" ]]; then echo "Release pull request contains non-metadata changes:" >&2 echo "${unexpected_files}" >&2 diff --git a/Cargo.lock b/Cargo.lock index 9af8992c37..8743bc865e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -616,26 +616,69 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] -name = "sqlbuild-rules-native" +name = "sqlbuild-analysis" version = "0.136.0" dependencies = [ - "fensu-policy", - "globset", - "mimalloc", "polyglot-sql", "polyglot-sql-function-catalogs", + "rayon", + "regex", + "serde", + "serde_json", + "sha2", + "sqlbuild-core", + "sqlbuild-sqltext", +] + +[[package]] +name = "sqlbuild-core" +version = "0.136.0" + +[[package]] +name = "sqlbuild-python" +version = "0.136.0" +dependencies = [ + "fensu-policy", + "mimalloc", "pyo3", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-core", + "sqlbuild-rules", + "sqlbuild-sqltext", +] + +[[package]] +name = "sqlbuild-rules" +version = "0.136.0" +dependencies = [ + "fensu-policy", + "globset", + "polyglot-sql", "rayon", "regex", "serde", "serde_json", "sha2", + "sqlbuild-analysis", + "sqlbuild-core", + "sqlbuild-sqltext", "sqlparser", "stacker", "tempfile", "toml", ] +[[package]] +name = "sqlbuild-sqltext" +version = "0.136.0" +dependencies = [ + "rayon", + "serde", + "sqlbuild-core", + "sqlparser", +] + [[package]] name = "sqlparser" version = "0.58.0" diff --git a/Cargo.toml b/Cargo.toml index 5dc0f8aca9..6f4c0756bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,29 @@ [workspace] members = [ - "crates/sqlbuild-rules-native", + "crates/sqlbuild-core", + "crates/sqlbuild-sqltext", + "crates/sqlbuild-analysis", + "crates/sqlbuild-rules", + "crates/sqlbuild-python", ] resolver = "2" +# Native crates from the bottom up: each crate may depend only on crates listed before it. +# `make check-ci` enforces this, keeps PyO3 in the Python boundary crate and keeps polyglot +# out of every crate below the polyglot floor. See crates/README.md. +[workspace.metadata.native-layers] +order = [ + "sqlbuild-core", + "sqlbuild-sqltext", + "sqlbuild-analysis", + "sqlbuild-rules", + "sqlbuild-python", +] +python-boundary = "sqlbuild-python" +polyglot-floor = "sqlbuild-analysis" + [workspace.package] +version = "0.136.0" edition = "2024" license = "Apache-2.0" publish = false @@ -36,8 +55,13 @@ quote = "1" rayon = "1" regex = "1" serde = { version = "1", features = ["derive"] } -serde_json = "1" +# Every crate relies on insertion-ordered JSON objects, which the extension has always built with. +serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" +sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } +sqlbuild-core = { path = "crates/sqlbuild-core" } +sqlbuild-rules = { path = "crates/sqlbuild-rules" } +sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } stacker = "0.1" sqlparser = { version = "0.58", features = ["visitor"] } syn = { version = "2", features = ["full", "visit", "extra-traits"] } diff --git a/Makefile b/Makefile index 38d0f4e140..f37d3a12ba 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check \ +.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering \ check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ @@ -566,7 +566,7 @@ verify-quick: exit $$status -check-ci: check-e2e-shards +check-ci: check-e2e-shards check-native-layering uv run ruff format --check . uv run ruff check . uv run ty check src tests @@ -574,6 +574,10 @@ check-ci: check-e2e-shards uv run fensu check +check-native-layering: + uv run python -m scripts.check_native_layering + + check-pr-metadata: @test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; } @test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; } diff --git a/crates/README.md b/crates/README.md new file mode 100644 index 0000000000..4bc3c8c67c --- /dev/null +++ b/crates/README.md @@ -0,0 +1,48 @@ +# Native crates + +SQLBuild's native code is a Cargo workspace that builds one Python extension module, +`sqlbuild._native`. Only `sqlbuild-python` knows about Python; every other crate is plain Rust +that can be tested and benchmarked with `cargo test` alone. + +| Crate | Responsibility | +|---|---| +| `sqlbuild-core` | Constants shared by several crates and the panic boundary that turns an unwinding native failure into a named compiler error. | +| `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution and static reference extraction. | +| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | +| `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | +| `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | + +## Layering + +Dependencies point one way, from the top of this graph to the bottom: + +```text +sqlbuild-python + -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-sqltext -> sqlbuild-core +``` + +Each crate may depend only on crates below it, and may also skip layers. The order is declared +in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and +`make check-native-layering` (part of `make check-ci`) fails when: + +- a crate depends on a crate that is not in an earlier layer, or a workspace crate is missing + from the order; +- any crate other than `sqlbuild-python` reaches `pyo3`, directly or through another package; +- a crate below `sqlbuild-analysis` reaches polyglot. + +The same check fails when the workspace version, the `pyproject.toml` version and +`.release-please-manifest.json` disagree, because the version is part of the build identity. + +`fensu check` applies the Rust structure rules inside each crate: other crates use items through +`main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`. + +## Working on the crates + +- Tests live next to the module they cover; run them for one crate with + `cargo test -p ` or for all crates with `cargo test --workspace`. +- `sqlbuild-analysis` has a `test-support` feature that exposes test-only helpers to the tests of + dependent crates. Enable it only from `[dev-dependencies]`. +- The build identity (`sqlbuild._native.BUILD_IDENTITY`) hashes the sources of every crate in + this directory, so editing any crate invalidates the native caches keyed on it. +- maturin builds `crates/sqlbuild-python`; the release version lives in + `[workspace.package]` and is shared by every crate. diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml new file mode 100644 index 0000000000..215d01af4c --- /dev/null +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "sqlbuild-analysis" +version.workspace = true +description = "Polyglot-based SQL analysis, binding and semantic validation for SQLBuild." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[features] +default = [] +# Expose test-only helpers to the tests of dependent crates. +test-support = [] + +[dependencies] +polyglot-sql.workspace = true +polyglot-sql-function-catalogs.workspace = true +rayon.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlbuild-core.workspace = true +sqlbuild-sqltext.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-rules-native/function_names/snowflake.txt b/crates/sqlbuild-analysis/function_names/snowflake.txt similarity index 100% rename from crates/sqlbuild-rules-native/function_names/snowflake.txt rename to crates/sqlbuild-analysis/function_names/snowflake.txt diff --git a/crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs b/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/column_references/_helpers/references.rs rename to crates/sqlbuild-analysis/src/column_references/_helpers/references.rs diff --git a/crates/sqlbuild-analysis/src/column_references/main/analyze.rs b/crates/sqlbuild-analysis/src/column_references/main/analyze.rs new file mode 100644 index 0000000000..c93e85e928 --- /dev/null +++ b/crates/sqlbuild-analysis/src/column_references/main/analyze.rs @@ -0,0 +1,3 @@ +pub fn analyze_json(request_json: &str) -> Result { + crate::column_references::references::analyze_json(request_json) +} diff --git a/crates/sqlbuild-analysis/src/column_references/main/mod.rs b/crates/sqlbuild-analysis/src/column_references/main/mod.rs new file mode 100644 index 0000000000..92d19eb519 --- /dev/null +++ b/crates/sqlbuild-analysis/src/column_references/main/mod.rs @@ -0,0 +1 @@ +pub mod analyze; diff --git a/crates/sqlbuild-rules-native/src/column_references/mod.rs b/crates/sqlbuild-analysis/src/column_references/mod.rs similarity index 78% rename from crates/sqlbuild-rules-native/src/column_references/mod.rs rename to crates/sqlbuild-analysis/src/column_references/mod.rs index c776b9c603..2236ec25e2 100644 --- a/crates/sqlbuild-rules-native/src/column_references/mod.rs +++ b/crates/sqlbuild-analysis/src/column_references/mod.rs @@ -1,4 +1,4 @@ -pub(crate) mod main; +pub mod main; #[path = "_helpers/references.rs"] mod references; diff --git a/crates/sqlbuild-rules-native/src/column_references/tests.rs b/crates/sqlbuild-analysis/src/column_references/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/column_references/tests.rs rename to crates/sqlbuild-analysis/src/column_references/tests.rs diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs b/crates/sqlbuild-analysis/src/column_references/tests/helpers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/column_references/tests/helpers.rs rename to crates/sqlbuild-analysis/src/column_references/tests/helpers.rs diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs b/crates/sqlbuild-analysis/src/column_references/tests/test_column_references.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/column_references/tests/test_column_references.rs rename to crates/sqlbuild-analysis/src/column_references/tests/test_column_references.rs diff --git a/crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs b/crates/sqlbuild-analysis/src/column_references/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/column_references/tests/test_types.rs rename to crates/sqlbuild-analysis/src/column_references/tests/test_types.rs diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs new file mode 100644 index 0000000000..0b35188f4d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs @@ -0,0 +1 @@ +pub mod sql_tests; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_rename.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_rename.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_rename.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_rename.rs index b3981e8258..92833a274b 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_rename.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_rename.rs @@ -3,9 +3,9 @@ use crate::compiler::_helpers::sql_tests::cte_slices::{ SliceDialect, WithSlices, opaque_end, read_identifier, }; -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::Unclosed; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; +use sqlbuild_sqltext::sql_scan::models::Unclosed; const DOT: &str = "."; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_slices.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_slices.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_slices.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_slices.rs index ba2f6f65ab..a914a9b88b 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_slices.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_slices.rs @@ -3,11 +3,11 @@ use std::collections::HashSet; use crate::compiler::_helpers::sql_tests::cte_sql::keyword_end; -use crate::rules::main::quote_policy::quote_policy; -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::{QuotePolicy, Unclosed}; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; +use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; +use sqlbuild_sqltext::sql_scan::models::{QuotePolicy, Unclosed}; /// Dialect lexical rules for slicing authored SQL. #[derive(Clone, Copy, Debug)] diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_sql.rs similarity index 95% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_sql.rs index eb335c5777..9ab3edb795 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/cte_sql.rs @@ -1,7 +1,7 @@ //! CTE definition and leading-WITH assembly shared by SQL-test planning and rendering. -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::skip_whitespace::skip_whitespace; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; /// Render one `name AS (body)` definition, closing a trailing line comment first. pub(crate) fn cte_definition_sql(name: &str, sql: &str) -> String { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/expected_columns.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index dd207b475b..ba6c8cfbd3 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -7,13 +7,13 @@ use serde::{Deserialize, Serialize}; use crate::constants::{ DIRECT_DEPENDENCY_PATH_LENGTH, MACRO_TEST_MODE, TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE, }; -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::main::quote_end::quote_end; -use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::QuotePolicy; -use crate::sql_scan::models::Unclosed; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::main::quote_end::quote_end; +use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; +use sqlbuild_sqltext::sql_scan::models::Unclosed; const MODEL_PREFIXES: [&str; 8] = [ "__macro__", diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs similarity index 94% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs index 362a9f285b..0e91901fac 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs @@ -5,10 +5,10 @@ use std::collections::HashSet; use regex::{Captures, Regex}; use crate::compiler::_helpers::sql_tests::planning::compile_error; -use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren; -use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; -use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::{LexicalSyntax, Unclosed}; +use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; +use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; pub(crate) fn replace_callable_markers( sql: &str, diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs new file mode 100644 index 0000000000..90b14e37ba --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -0,0 +1,10 @@ +pub mod cte_rename; +pub mod cte_slices; +pub mod cte_sql; +pub mod expected_columns; +pub mod extraction; +pub mod helper_scope; +pub mod markers; +pub mod planning; +pub mod relation_markers; +pub mod rendering; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 7f83a43a60..0dfeae02ca 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -27,7 +27,7 @@ use crate::compiler::_helpers::sql_tests::rendering::{ rendered_chain_steps, }; use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE}; -use crate::sql_scan::models::LexicalSyntax; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/relation_markers.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs rename to crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs new file mode 100644 index 0000000000..b45f3bacd3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -0,0 +1,6 @@ +pub mod sql_test_chain_resolution; +pub mod sql_test_difference_sampling; +pub mod sql_test_extraction; +pub mod sql_test_fixture_facts; +pub mod sql_test_planning; +pub mod sql_test_rendering; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_chain_resolution.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chain_resolution.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_chain_resolution.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_chain_resolution.rs index 4e43c334a8..74ae93cc0d 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_chain_resolution.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chain_resolution.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers; -pub(crate) fn resolve_chains_json(request_json: &str) -> Result { +pub fn resolve_chains_json(request_json: &str) -> Result { _helpers::sql_tests::planning::resolve_chains_json(request_json) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_difference_sampling.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_difference_sampling.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_difference_sampling.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_difference_sampling.rs index a3fa1a344d..c86490d1a6 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_difference_sampling.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_difference_sampling.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers; -pub(crate) fn render_difference_sample_json(request_json: &str) -> Result { +pub fn render_difference_sample_json(request_json: &str) -> Result { _helpers::sql_tests::rendering::render_difference_sample_json(request_json) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_extraction.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_extraction.rs similarity index 56% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_extraction.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_extraction.rs index 4642e447a2..b125f0715e 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_extraction.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers; -pub(crate) fn extract_batch_json(request_json: &str) -> Result { +pub fn extract_batch_json(request_json: &str) -> Result { _helpers::sql_tests::extraction::extract_batch_json(request_json) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_fixture_facts.rs similarity index 84% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_fixture_facts.rs index 95d6f216c7..4f438be8e7 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_fixture_facts.rs @@ -1,7 +1,7 @@ use crate::compiler::_helpers::sql_tests::{extraction, planning}; use crate::compiler::models::SqlTestFixtureFacts; -pub(crate) fn fixture_facts(name: &str, sql: &str) -> SqlTestFixtureFacts { +pub fn fixture_facts(name: &str, sql: &str) -> SqlTestFixtureFacts { SqlTestFixtureFacts { mock: planning::MOCK_CTE_PREFIXES.iter().any(|prefix| { name.strip_prefix(prefix) diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_planning.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_planning.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_planning.rs index 0bbbe36efd..d5057b8241 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_planning.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers; -pub(crate) fn plan_and_render_json(request_json: &str) -> Result { +pub fn plan_and_render_json(request_json: &str) -> Result { _helpers::sql_tests::planning::plan_and_render_json(request_json) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_rendering.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_rendering.rs similarity index 56% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_test_rendering.rs rename to crates/sqlbuild-analysis/src/compiler/main/sql_test_rendering.rs index a7a59c625b..ec61e53e71 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_rendering.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_rendering.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers; -pub(crate) fn render_json(request_json: &str) -> Result { +pub fn render_json(request_json: &str) -> Result { _helpers::sql_tests::rendering::render_json(request_json) } diff --git a/crates/sqlbuild-analysis/src/compiler/mod.rs b/crates/sqlbuild-analysis/src/compiler/mod.rs new file mode 100644 index 0000000000..6059132c3b --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/mod.rs @@ -0,0 +1,6 @@ +pub mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/compiler/models.rs b/crates/sqlbuild-analysis/src/compiler/models.rs new file mode 100644 index 0000000000..bef142348d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/models.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SqlTestFixtureFacts { + pub mock: bool, + pub empty_fixture_marker: bool, +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs similarity index 70% rename from crates/sqlbuild-rules-native/src/compiler/tests.rs rename to crates/sqlbuild-analysis/src/compiler/tests.rs index 85fc3262d1..b06523a89f 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,13 +1,5 @@ #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_model_header_matching.rs"] -mod model_header_matching; -#[path = "tests/test_model_header_tokenization.rs"] -mod model_header_tokenization; -#[path = "tests/test_sql_interpolation.rs"] -mod sql_interpolation; -#[path = "tests/test_sql_references.rs"] -mod sql_references; #[path = "tests/test_sql_test_cte_rename.rs"] mod sql_test_cte_rename; #[path = "tests/test_sql_test_cte_slices.rs"] diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs rename to crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index 9d721acaec..e1b200fdb9 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -5,215 +5,8 @@ use std::sync::{Arc, Barrier, Mutex}; use polyglot_sql::Dialect; use serde_json::{Value, json}; -use crate::compiler::_helpers::model_headers::tokenization::{ - MAX_TOKENIZER_WORKERS, TOKENIZER_WORKER_STACK_BYTES, build_tokenizer_pool, parse_batch, -}; -use crate::compiler::_helpers::sql_interpolation::substitution::{ - FALLBACK, SUBSTITUTED, UNCHANGED, substitute_batch, -}; -use crate::compiler::_helpers::sql_references::extraction::extract; use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; use crate::compiler::main::sql_test_extraction::extract_batch_json; -use crate::compiler::models::AuthoredValue; - -pub(crate) fn scalar_variables_preserve_lexical_boundaries() -> bool { - let sqls = vec![ - "SELECT @@revision, '@@status', @@@window_start".to_owned(), - "-- @@revision\nSELECT /* @@status */ 1".to_owned(), - "SELECT '@@revision''s'".to_owned(), - ]; - substitute_batch( - &sqls, - &[ - ("revision".to_owned(), "7".to_owned()), - ("status".to_owned(), "ready".to_owned()), - ], - ) == vec![ - ( - SUBSTITUTED, - Some("SELECT 7, 'ready', @@@window_start".to_owned()), - ), - (UNCHANGED, None), - (SUBSTITUTED, Some("SELECT '7''s'".to_owned())), - ] -} - -pub(crate) fn dynamic_or_malformed_sql_requests_fallback() -> bool { - let sqls = vec![ - "SELECT @@ENV:USER".to_owned(), - "SELECT @@missing".to_owned(), - "SELECT @@revision, 'unterminated".to_owned(), - "SELECT @@revision /* unterminated".to_owned(), - "SELECT @@révision".to_owned(), - ]; - substitute_batch(&sqls, &[("revision".to_owned(), "7".to_owned())]) - == vec![(FALLBACK, None); sqls.len()] -} - -pub(crate) fn simple_references_preserve_authored_order() -> bool { - extract( - "SELECT * FROM __source('orders') UNION ALL SELECT * FROM __dbt_ref(\"shop\", \"customers\")", - ) == Some(vec![ - ("source".to_owned(), "orders".to_owned(), None, None), - ( - "dbt_ref".to_owned(), - "customers".to_owned(), - Some("shop".to_owned()), - None, - ), - ]) -} - -pub(crate) fn comments_and_quoted_text_hide_references() -> bool { - extract("-- __ref(\"ignored\")\nSELECT '__seed(\"also_ignored\")' FROM __ref(orders)") - == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) -} - -pub(crate) fn dollar_quoted_text_hides_references() -> bool { - extract(concat!( - "SELECT $$Customer's order -- __ref(\"ignored\") $5$$ AS label, ", - "$tag$ $$ __seed(\"also_ignored\") $tag$ AS note, price$1$ ", - "FROM __ref(orders)" - )) == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) -} - -pub(crate) fn complex_or_malformed_sql_requests_fallback() -> bool { - [ - "SELECT * FROM __table_fn(\"orders\")(1)", - "SELECT * FROM __ref(concat('ord', 'ers'))", - "SELECT * FROM __ref(\"orders\"", - "SELECT * FROM __ref(\"orders\") /* unterminated", - "SELECT * FROM __ref(\"orders\") WHERE note = 'unterminated", - "SELECT * FROM __ref(\"orders\") WHERE note = $$unterminated", - "SELECT * FROM __ref(örders)", - ] - .into_iter() - .all(|sql| extract(sql).is_none()) -} - -pub(crate) fn nested_authored_headers_preserve_values_and_offsets() -> bool { - let headers = vec![ - " columns (café (type DECIMAL(10,2))), enabled true".to_owned(), - "constants (_countries {FR, GB}, _size constant(value 2))".to_owned(), - ]; - let results = parse_batch(&headers).expect("worker pool builds"); - assert_eq!(results[0].2, None); - assert_eq!( - results[0].1.as_ref().expect("column offsets"), - &vec![("café".to_owned(), 10, 4)] - ); - assert_eq!( - results[0].0, - Some(AuthoredValue::Map(vec![ - ( - "columns".to_owned(), - AuthoredValue::Map(vec![( - "café".to_owned(), - AuthoredValue::Map(vec![( - "type".to_owned(), - AuthoredValue::String("DECIMAL(10,2)".to_owned()), - )]), - )]), - ), - ("enabled".to_owned(), AuthoredValue::Boolean(true)), - ])) - ); - assert!(matches!( - results[1].0, - Some(AuthoredValue::Map(ref values)) if values.len() == 1 - )); - true -} - -pub(crate) fn invalid_headers_return_each_exact_error_in_order() -> bool { - let headers = vec![ - "schema \"analytics".to_owned(), - "schema analytics\"mart".to_owned(), - "schema ${ENV".to_owned(), - ]; - let results = parse_batch(&headers).expect("worker pool builds"); - assert_eq!( - results - .iter() - .map(|result| result.2.as_deref()) - .collect::>(), - vec![ - Some("unterminated double-quoted string at position 7"), - Some("unexpected double quote inside bare value at position 16; quote the whole value"), - Some("unterminated template value at position 7"), - ] - ); - true -} - -pub(crate) fn header_comments_are_skipped_without_shifting_offsets() -> bool { - let headers = vec![ - "materialized table -- keep model metadata\n columns (id ()) /* it's, (fine) */" - .to_owned(), - "materialized table--trailing note\n".to_owned(), - "materialized table /* unclosed".to_owned(), - ]; - - let results = parse_batch(&headers).expect("worker pool builds"); - - let AuthoredValue::Map(values) = results[0].0.as_ref().expect("header values") else { - panic!("header values must be a map"); - }; - assert_eq!( - values - .iter() - .map(|(key, _)| key.as_str()) - .collect::>(), - vec!["materialized", "columns"] - ); - assert_eq!( - results[0].1.as_ref().expect("column offsets"), - &vec![("id".to_owned(), 53, 2)] - ); - let AuthoredValue::Map(values) = results[1].0.as_ref().expect("header values") else { - panic!("header values must be a map"); - }; - assert_eq!( - values - .iter() - .map(|(key, _)| key.as_str()) - .collect::>(), - vec!["materialized"] - ); - assert_eq!( - results[2].2.as_deref(), - Some("unterminated block comment at position 19") - ); - true -} - -pub(crate) fn nested_and_root_columns_return_only_root_offsets() -> bool { - let headers = vec![ - "config (columns (nested (type INTEGER))), columns (top (type INTEGER))".to_owned(), - "config (columns (nested (type INTEGER)))".to_owned(), - ]; - - let results = parse_batch(&headers).expect("worker pool builds"); - - assert_eq!( - results[0].1.as_ref().expect("column offsets"), - &vec![("top".to_owned(), 51, 3)] - ); - assert_eq!( - results[1].1.as_ref().expect("column offsets"), - &Vec::<(String, usize, usize)>::new() - ); - true -} - -pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { - for (header_count, expected_workers) in [(0, 1), (1, 1), (3, 3), (5, MAX_TOKENIZER_WORKERS)] { - let pool = build_tokenizer_pool(header_count).expect("worker pool builds"); - assert_eq!(pool.current_num_threads(), expected_workers); - } - assert_eq!(TOKENIZER_WORKER_STACK_BYTES, 16 * 1024 * 1024); - true -} pub(crate) fn mixed_expanded_tests_preserve_order_and_payloads() -> bool { let response = extract_batch_json(r#"{"tests":[{"sql":"WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), __expected__orders AS (SELECT id FROM helper), __assert__positive AS (SELECT id FROM helper WHERE id < 0) SELECT 1","fileLabel":"tests/orders.sql","mode":"model"},{"sql":"WITH input AS (SELECT 1 AS value), __udf_actual__ AS (SELECT __udf(\"increment\")(value) AS value FROM input), __udf_expected__ AS (SELECT 2 AS value) SELECT 1","fileLabel":"tests/increment.sql","mode":"udf"}]}"#).expect("batch succeeds"); diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_cte_rename.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_cte_rename.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_cte_rename.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_cte_rename.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_cte_slices.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_cte_slices.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_cte_slices.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_cte_slices.rs index e3d1b96c66..2be5fe53fd 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_cte_slices.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_cte_slices.rs @@ -1,6 +1,6 @@ use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, split_top_level_with}; use crate::compiler::tests::test_types::CteSliceTestCase; -use crate::sql_scan::models::Unclosed; +use sqlbuild_sqltext::sql_scan::models::Unclosed; #[test] fn given_leading_with_queries_when_splitting_then_authored_slices_or_none_are_returned() { diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_expected_columns.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_markers.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs index 17f55cbbfc..281b952c67 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs @@ -4,7 +4,7 @@ use crate::compiler::_helpers::sql_tests::markers::{ in_protected_range, marker_names, replace_named_markers, }; use crate::compiler::tests::test_types::{MarkerNamesTestCase, ProtectedRangeTestCase}; -use crate::sql_scan::models::LexicalSyntax; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; #[test] fn given_markers_around_comments_and_strings_when_scanning_then_only_code_markers_are_used() { diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_relation_markers.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_rendering.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_rendering.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs similarity index 80% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs rename to crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index bf566c43ff..b42cb7e7ed 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -1,16 +1,4 @@ -use crate::sql_scan::models::Unclosed; - -pub(crate) struct ModelHeaderTokenizationTestCase { - pub(crate) description: &'static str, - pub(crate) run: fn() -> bool, - pub(crate) expected_success: bool, -} - -pub(crate) struct StaticSqlOperationTestCase { - pub(crate) description: &'static str, - pub(crate) run: fn() -> bool, - pub(crate) expected_success: bool, -} +use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(crate) struct SqlTestExtractionTestCase { pub(crate) description: &'static str, @@ -86,9 +74,3 @@ pub(crate) struct RelationMarkerOracleTestCase { pub(crate) dialect: &'static str, pub(crate) expected_mismatches: &'static [&'static str], } - -pub(crate) struct ModelHeaderMatchTestCase { - pub(crate) description: &'static str, - pub(crate) contents: &'static str, - pub(crate) expected_offsets: Option<(usize, usize, usize)>, -} diff --git a/crates/sqlbuild-analysis/src/constants.rs b/crates/sqlbuild-analysis/src/constants.rs new file mode 100644 index 0000000000..67a67562e8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/constants.rs @@ -0,0 +1,11 @@ +pub(crate) const DATETIME_TYPE: &str = "DATETIME"; +pub(crate) const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; +pub(crate) const MACRO_TEST_MODE: &str = "macro"; +pub(crate) const QUOTED_IDENTIFIER_DELIMITER_BYTES: usize = 2; +pub(crate) const SQL_TEST_ACTUAL_CTE: &str = "__actual"; +pub(crate) const SQL_TEST_EXPECTED_CTE: &str = "__expected"; +pub(crate) const SQL_TEST_ACTUAL_CTE_PREFIX: &str = "__actual__"; +pub(crate) const TABLE_FUNCTION_TEST_MODE: &str = "table_fn"; +pub(crate) const UDF_TEST_MODE: &str = "udf"; +pub(crate) const UNKNOWN_SQL_TYPE: &str = "UNKNOWN"; +pub(crate) const VARCHAR_SQL_TYPE: &str = "VARCHAR"; diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs new file mode 100644 index 0000000000..74d7298030 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -0,0 +1,11 @@ +//! SQL analysis over polyglot: query facts, binding catalog, semantic validation and SQL tests. + +#![forbid(unsafe_code)] + +pub mod column_references; +pub mod compiler; +mod constants; +pub mod query_analysis; +pub mod semantic_usage; +pub mod semantic_validation; +pub mod sql_tokens; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/borrowed_facts.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/borrowed_facts.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/_helpers/borrowed_facts.rs rename to crates/sqlbuild-analysis/src/query_analysis/_helpers/borrowed_facts.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/_helpers/compatibility_types.rs rename to crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs rename to crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index 1f95a3bc07..b5def1c23c 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -12,7 +12,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; -use crate::constants::{SQL_WILDCARD, UNKNOWN_SQL_TYPE, VARCHAR_SQL_TYPE}; +use crate::constants::{UNKNOWN_SQL_TYPE, VARCHAR_SQL_TYPE}; +use sqlbuild_core::constants::SQL_WILDCARD; const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs new file mode 100644 index 0000000000..c1c36ff2f0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs @@ -0,0 +1,3 @@ +pub fn analyze_json(request_json: &str) -> Result { + crate::query_analysis::engine::analyze_json(request_json) +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs new file mode 100644 index 0000000000..50f36ef4ae --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs @@ -0,0 +1,3 @@ +pub fn analyze_project_json(request_json: &str) -> Result { + crate::query_analysis::engine::analyze_project_json(request_json) +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs new file mode 100644 index 0000000000..5e83be1b56 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs @@ -0,0 +1,3 @@ +pub fn analyze_project_compact_json(request_json: &str) -> Result { + crate::query_analysis::engine::analyze_project_compact_json(request_json) +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs new file mode 100644 index 0000000000..3e02e0d854 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -0,0 +1,4 @@ +pub mod analyze; +pub mod analyze_project; +pub mod analyze_project_compact; +pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/prepare_project_catalog.rs b/crates/sqlbuild-analysis/src/query_analysis/main/prepare_project_catalog.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/main/prepare_project_catalog.rs rename to crates/sqlbuild-analysis/src/query_analysis/main/prepare_project_catalog.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/mod.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/query_analysis/mod.rs rename to crates/sqlbuild-analysis/src/query_analysis/mod.rs index 6350d84af2..94f93ea625 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/mod.rs @@ -4,7 +4,7 @@ mod borrowed_facts; mod compatibility_types; #[path = "_helpers/engine.rs"] mod engine; -pub(crate) mod main; +pub mod main; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests.rs b/crates/sqlbuild-analysis/src/query_analysis/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/tests.rs rename to crates/sqlbuild-analysis/src/query_analysis/tests.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs b/crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs rename to crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs b/crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs rename to crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/test_types.rs b/crates/sqlbuild-analysis/src/query_analysis/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/query_analysis/tests/test_types.rs rename to crates/sqlbuild-analysis/src/query_analysis/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/semantic_usage/main.rs rename to crates/sqlbuild-analysis/src/semantic_usage/main.rs index 0f5aa89fac..37a3d7e161 100644 --- a/crates/sqlbuild-rules-native/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -46,7 +46,7 @@ fn default_dialect() -> String { "generic".to_string() } -pub(crate) fn analyze_json(request_json: &str) -> Result { +pub fn analyze_json(request_json: &str) -> Result { let request: UsageRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; let dialect: DialectType = request.dialect.parse().map_err(|error| { diff --git a/crates/sqlbuild-rules-native/src/semantic_usage/mod.rs b/crates/sqlbuild-analysis/src/semantic_usage/mod.rs similarity index 54% rename from crates/sqlbuild-rules-native/src/semantic_usage/mod.rs rename to crates/sqlbuild-analysis/src/semantic_usage/mod.rs index c90d047f0b..e29b572755 100644 --- a/crates/sqlbuild-rules-native/src/semantic_usage/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/mod.rs @@ -1,4 +1,4 @@ -pub(crate) mod main; +pub mod main; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/semantic_usage/tests.rs b/crates/sqlbuild-analysis/src/semantic_usage/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_usage/tests.rs rename to crates/sqlbuild-analysis/src/semantic_usage/tests.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_usage/tests/test_semantic_usage.rs b/crates/sqlbuild-analysis/src/semantic_usage/tests/test_semantic_usage.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_usage/tests/test_semantic_usage.rs rename to crates/sqlbuild-analysis/src/semantic_usage/tests/test_semantic_usage.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_usage/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_usage/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_usage/tests/test_types.rs rename to crates/sqlbuild-analysis/src/semantic_usage/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs similarity index 72% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index a44bde62ff..2c2e139df8 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -1,8 +1,5 @@ //! Compile-owned schema catalog and native, batched binding requests. -use crate::bindings::main::compiler_error::compiler_error; -use crate::bindings::main::normalization_results::normalization_results; -use crate::bindings::types::CompilerDetach; use crate::semantic_validation::main::normalize_batch; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -12,7 +9,7 @@ use crate::semantic_validation::models::{ CatalogInput, Columns, CompactAnalysisJob, FunctionProbes, ProjectCatalog, }; use crate::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, Relations, + BindingRequest, DiagnosticRow, NormalizationRequest, PreparedCompactAnalysis, Relations, }; use crate::semantic_validation::{ _helpers::{diagnostics, identifiers}, @@ -23,88 +20,69 @@ use polyglot_sql::{ Dialect, DialectType, SchemaValidationOptions, ValidationError, ValidationResult, ValidationSchema, }; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, FromPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pymethods; -use pyo3::types::{PyBytes, PyDict, PyDictMethods}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; const ANALYSIS_WORKERS: usize = 4; const ANALYSIS_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; -impl<'py> FromPyObject<'py> for Columns { - fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { - value - .downcast::()? - .iter() - .map(|(name, data_type)| Ok((name.extract()?, data_type.extract()?))) - .collect::>>() - .map(Self) - } -} -#[pymethods] impl ProjectCatalog { - fn inferred_schema( + /// Map inferred column types onto the binding names the query projects. + pub fn inferred_schema( &self, - py: Python<'_>, sql: &str, columns: Columns, inputs: Relations, - ) -> PyResult>> { - py.compiler_detach(|| { - use polyglot_sql::{Expression, ExpressionWalk}; - let parsed = Dialect::get(self.dialect) - .parse(sql) - .map_err(|error| error.to_string())?; - let mut identifiers: HashMap = HashMap::new(); - for expression in &parsed { - if let Some(select) = expression.dfs().find_map(|node| match node { - Expression::Select(select) => Some(select), - _ => None, - }) { - for projection in &select.expressions { - let identifier = match projection { - Expression::Alias(alias) => Some(&alias.alias), - Expression::Column(column) => Some(&column.name), - _ => None, + ) -> Result>, String> { + use polyglot_sql::{Expression, ExpressionWalk}; + let parsed = Dialect::get(self.dialect) + .parse(sql) + .map_err(|error| error.to_string())?; + let mut identifiers: HashMap = HashMap::new(); + for expression in &parsed { + if let Some(select) = expression.dfs().find_map(|node| match node { + Expression::Select(select) => Some(select), + _ => None, + }) { + for projection in &select.expressions { + let identifier = match projection { + Expression::Alias(alias) => Some(&alias.alias), + Expression::Column(column) => Some(&column.name), + _ => None, + }; + if let Some(identifier) = identifier { + let name = if identifier.quoted { + format!("\"{}\"", identifier.name.replace('"', "\"\"")) + } else { + identifier.name.clone() }; - if let Some(identifier) = identifier { - let name = if identifier.quoted { - format!("\"{}\"", identifier.name.replace('"', "\"\"")) - } else { - identifier.name.clone() - }; - identifiers.insert(identifier.name.clone(), name); - } + identifiers.insert(identifier.name.clone(), name); } - break; } + break; } - let mut exact_inputs: HashMap = HashMap::new(); - for shape in inputs.values() { - for (name, _) in &shape.0 { - if name.len() > 1 && name.starts_with('"') && name.ends_with('"') { - exact_inputs - .insert(name[1..name.len() - 1].replace("\"\"", "\""), name.clone()); - } + } + let mut exact_inputs: HashMap = HashMap::new(); + for shape in inputs.values() { + for (name, _) in &shape.0 { + if name.len() > 1 && name.starts_with('"') && name.ends_with('"') { + exact_inputs + .insert(name[1..name.len() - 1].replace("\"\"", "\""), name.clone()); } } - let mut result: HashMap> = HashMap::new(); - for (name, column_type) in columns.0 { - let binding_name = identifiers - .get(&name) - .or_else(|| exact_inputs.get(&name)) - .cloned() - .unwrap_or(name); - result.insert(binding_name, column_type); - } - Ok(result) - }) - .map_err(compiler_error) + } + let mut result: HashMap> = HashMap::new(); + for (name, column_type) in columns.0 { + let binding_name = identifiers + .get(&name) + .or_else(|| exact_inputs.get(&name)) + .cloned() + .unwrap_or(name); + result.insert(binding_name, column_type); + } + Ok(result) } - #[new] - fn new(request: CatalogInput) -> PyResult { + pub fn new(request: CatalogInput) -> Result { let CatalogInput { dialect, quoted_ignore_case, @@ -114,7 +92,7 @@ impl ProjectCatalog { } = request; let dialect = dialect .parse::() - .map_err(|error| PyValueError::new_err(error.to_string()))?; + .map_err(|error| error.to_string())?; let mut catalog = Self { dialect, quoted_ignore_case, @@ -148,13 +126,13 @@ impl ProjectCatalog { Ok(catalog) } - fn update_relations(&mut self, relations: Relations) { + pub fn update_relations(&mut self, relations: Relations) { for (name, columns) in relations { self.tables.insert(name.clone(), self.table(&name, columns)); } } - fn with_relations(&self, relations: Relations) -> Self { + pub fn with_relations(&self, relations: Relations) -> Self { let mut catalog = Self { dialect: self.dialect, options: self.options.clone(), @@ -170,7 +148,7 @@ impl ProjectCatalog { catalog } - fn update_analysis(&mut self, relations: HashMap) { + pub fn update_analysis(&mut self, relations: HashMap) { for (name, (types, nullability)) in relations { let types: HashMap<_, _> = types.0.into_iter().collect(); let mut table = self.table(&name, Columns::default()); @@ -196,7 +174,7 @@ impl ProjectCatalog { } } - fn register_override(&mut self, relations: Relations) -> usize { + pub fn register_override(&mut self, relations: Relations) -> usize { let tables: HashMap = relations .into_iter() .map(|(name, columns)| (name.clone(), self.table(&name, columns))) @@ -205,83 +183,66 @@ impl ProjectCatalog { self.overrides.len() - 1 } - fn validation_payload(&self, requests: Vec) -> PyResult { + pub fn validation_payload(&self, requests: Vec) -> Result { let mut payloads = Vec::with_capacity(requests.len()); for (sql, references, overrides) in requests { - let schema = self - .schema(&references, overrides) - .map_err(compiler_error)?; + let schema = self.schema(&references, overrides)?; payloads.push(serde_json::json!({"sql": sql, "dialect": self.dialect.to_string(), "schema": schema, "options": self.options, "quoted_ignore_case": self.quoted_ignore_case})); } - serde_json::to_string(&payloads).map_err(|error| PyValueError::new_err(error.to_string())) + serde_json::to_string(&payloads).map_err(|error| error.to_string()) } /// Resolve a compact batch now; the job runs later while the catalog keeps changing. - fn prepare_compact(&self, py: Python<'_>, payload: &[u8]) -> PyResult { - let text = std::str::from_utf8(payload) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - let analysis = py - .compiler_detach(|| { - crate::query_analysis::main::prepare_project_catalog::prepare_project_compact_with_catalog( - text, self, - ) - }) - .map_err(compiler_error)?; + pub fn prepare_compact(&self, text: &str) -> Result { + let analysis = + crate::query_analysis::main::prepare_project_catalog::prepare_project_compact_with_catalog( + text, self, + )?; Ok(CompactAnalysisJob { analysis: Mutex::new(Some(analysis)), catalog: self.analysis_view(), }) } - fn binding_results( + pub fn binding_results( &self, - py: Python<'_>, requests: Vec, - ) -> PyResult>> { - py.compiler_detach(|| { - let pool = self.analysis_pool()?; - let _running = RunningAnalysis::start(&self.running_analyses); - pool.install(|| { - requests - .into_par_iter() - .map(|(sql, references, overrides)| { - let schema = self.schema(&references, overrides)?; - let result = self.validate(&sql, &schema)?; - let result = diagnostics::map_diagnostics( - &sql, - self.dialect, - result, - &self.function_probes, - )?; - Ok(result.errors.into_iter().map(diagnostic_row).collect()) - }) - .collect::, String>>() - }) + ) -> Result>, String> { + let pool = self.analysis_pool()?; + let _running = RunningAnalysis::start(&self.running_analyses); + pool.install(|| { + requests + .into_par_iter() + .map(|(sql, references, overrides)| { + let schema = self.schema(&references, overrides)?; + let result = self.validate(&sql, &schema)?; + let result = diagnostics::map_diagnostics( + &sql, + self.dialect, + result, + &self.function_probes, + )?; + Ok(result.errors.into_iter().map(diagnostic_row).collect()) + }) + .collect::, String>>() }) - .map_err(compiler_error) } - /// Normalize a preparation batch on the idle analysis pool, else in turn, in one detached call. - fn normalize_analysis_sqls( + /// Normalize a preparation batch on the idle analysis pool, else in turn. + pub fn normalize_analysis_sqls( &self, - py: Python<'_>, dialect: &str, requests: Vec, - ) -> PyResult>> { - let results = py - .compiler_detach(|| { - let pool = (self.running_analyses.load(Ordering::Acquire) == 0) - .then(|| self.analysis_pool()) - .transpose()?; - Ok(normalize_batch::normalize_analysis_sqls( - dialect, - requests, - pool.as_deref(), - )) - }) - .map_err(compiler_error)?; - normalization_results(py, results) + ) -> Result>, String> { + let pool = (self.running_analyses.load(Ordering::Acquire) == 0) + .then(|| self.analysis_pool()) + .transpose()?; + Ok(normalize_batch::normalize_analysis_sqls( + dialect, + requests, + pool.as_deref(), + )) } } @@ -301,22 +262,19 @@ impl Drop for RunningAnalysis<'_> { } } -#[pymethods] impl CompactAnalysisJob { - fn run<'py>(&self, py: Python<'py>) -> PyResult> { - let analysis = self - .analysis + /// Take the resolved analysis; a job runs at most once. + pub fn take_analysis(&self) -> Option { + self.analysis .lock() .unwrap_or_else(PoisonError::into_inner) .take() - .ok_or_else(|| PyValueError::new_err("compact analysis job already ran"))?; - let result = py - .compiler_detach(|| { - let _running = RunningAnalysis::start(&self.catalog.running_analyses); - analysis(&self.catalog) - }) - .map_err(compiler_error)?; - Ok(PyBytes::new(py, result.as_bytes())) + } + + /// Run a taken analysis, counted as running on the catalog's pool. + pub fn run(&self, analysis: PreparedCompactAnalysis) -> Result { + let _running = RunningAnalysis::start(&self.catalog.running_analyses); + analysis(&self.catalog) } } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs index 858cf88e53..b1c0b34a6a 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs @@ -2,11 +2,11 @@ use crate::semantic_validation::models::FunctionProbes; use crate::semantic_validation::types::ProbeKey; -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::models::{QuotePolicy, Unclosed}; use polyglot_sql::{Dialect, DialectType, Expression, ValidationError}; use polyglot_sql_function_catalogs::{CatalogSink, FunctionNameCase, FunctionSignature}; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::{QuotePolicy, Unclosed}; use std::collections::{HashMap, HashSet}; use std::mem::Discriminant; use std::sync::{LazyLock, OnceLock, PoisonError}; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/identifiers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/identifiers.rs diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs new file mode 100644 index 0000000000..d9b898d3e4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub mod catalog; +pub mod diagnostics; +pub mod function_names; +pub mod identifiers; +pub mod normalization; +pub mod positions; +pub mod type_locations; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 1d60228f59..eb86232816 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -1,9 +1,9 @@ //! Analysis normalization with a character-offset provenance map. -use crate::sql_scan::main::matching_paren::matching_paren; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::models::QuotePolicy; use regex::Regex; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; use std::collections::HashMap; use std::sync::LazyLock; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs similarity index 68% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs index e207e6dcc2..d59718854b 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs @@ -1,11 +1,8 @@ //! Native authored positions, including macro-expansion passes. -use crate::bindings::main::compiler_guard::compiler_guard; use crate::semantic_validation::_helpers::diagnostics::column_pattern; use crate::semantic_validation::_helpers::normalization::normalize; use crate::semantic_validation::models::{BindingPositions, PositionInput}; -use pyo3::exceptions::PyValueError; -use pyo3::{PyResult, pymethods}; use regex::Regex; use std::collections::HashMap; use std::sync::{LazyLock, OnceLock}; @@ -15,92 +12,83 @@ static TOKENS: LazyLock> = LazyLock::new(|| Regex::new(r"\w+|[^\w\s]").map_err(|error| error.to_string())); type Mapping = (usize, usize, usize, usize); -#[pymethods] impl BindingPositions { - #[new] - fn new(request: PositionInput) -> PyResult { - compiler_guard(|| { - let PositionInput { - authored, - query, - expanded, - cleaned, - passes, - } = request; - let mapped = normalize(&expanded, "snowflake", &HashMap::new(), &HashMap::new()); - let offsets = match mapped { - Ok(mapped) if mapped.sql == cleaned => mapped.offsets, - _ if expanded == cleaned => (0..=expanded.chars().count()).collect(), - _ => align_offsets(&expanded, &cleaned).map_err(PyValueError::new_err)?, - }; - Ok(Self { - query_start: authored - .find(&query) - .map(|offset| authored[..offset].chars().count()), - lines: line_starts(&authored), - cleaned_lines: line_starts(&cleaned), - authored, - offsets, - passes, - words: OnceLock::new(), - }) + pub fn new(request: PositionInput) -> Result { + let PositionInput { + authored, + query, + expanded, + cleaned, + passes, + } = request; + let mapped = normalize(&expanded, "snowflake", &HashMap::new(), &HashMap::new()); + let offsets = match mapped { + Ok(mapped) if mapped.sql == cleaned => mapped.offsets, + _ if expanded == cleaned => (0..=expanded.chars().count()).collect(), + _ => align_offsets(&expanded, &cleaned)?, + }; + Ok(Self { + query_start: authored + .find(&query) + .map(|offset| authored[..offset].chars().count()), + lines: line_starts(&authored), + cleaned_lines: line_starts(&cleaned), + authored, + offsets, + passes, + words: OnceLock::new(), }) } - fn position( + pub fn position( &self, start: Option, line: Option, column: Option, message: &str, - ) -> PyResult<(Option, Option)> { - compiler_guard(|| { - if let Some(captures) = column_pattern() - .map_err(PyValueError::new_err)? - .captures(message) - { - let identifier = captures[1].rsplit('.').next().unwrap_or(&captures[1]); - if let Some(offset) = self.unique_word_offset(identifier) { - return Ok(position(&self.lines, offset)); - } + ) -> Result<(Option, Option), String> { + if let Some(captures) = column_pattern()?.captures(message) { + let identifier = captures[1].rsplit('.').next().unwrap_or(&captures[1]); + if let Some(offset) = self.unique_word_offset(identifier) { + return Ok(position(&self.lines, offset)); } - let Some(query_start) = self.query_start else { - return Ok((None, None)); - }; - let offset = start.or_else(|| { - line.zip(column).map(|(line, column)| { - self.cleaned_lines - .get(line.saturating_sub(1)) - .copied() - .unwrap_or(0) - + column.saturating_sub(1) - }) - }); - let Some(offset) = offset else { - return Ok(position(&self.lines, query_start)); - }; - let mut mapped = self - .offsets - .get(offset) - .copied() - .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); - for pass in self.passes.iter().rev() { - let mut adjusted = mapped as isize; - for &(source_start, source_end, output_start, output_end) in pass { - if mapped < output_start { - break; - } - if mapped < output_end { - adjusted = source_start as isize; - break; - } - adjusted += - (source_end - source_start) as isize - (output_end - output_start) as isize; + } + let Some(query_start) = self.query_start else { + return Ok((None, None)); + }; + let offset = start.or_else(|| { + line.zip(column).map(|(line, column)| { + self.cleaned_lines + .get(line.saturating_sub(1)) + .copied() + .unwrap_or(0) + + column.saturating_sub(1) + }) + }); + let Some(offset) = offset else { + return Ok(position(&self.lines, query_start)); + }; + let mut mapped = self + .offsets + .get(offset) + .copied() + .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); + for pass in self.passes.iter().rev() { + let mut adjusted = mapped as isize; + for &(source_start, source_end, output_start, output_end) in pass { + if mapped < output_start { + break; + } + if mapped < output_end { + adjusted = source_start as isize; + break; } - mapped = adjusted.max(0) as usize; + adjusted += + (source_end - source_start) as isize - (output_end - output_start) as isize; } - Ok(position(&self.lines, query_start + mapped)) - }) + mapped = adjusted.max(0) as usize; + } + Ok(position(&self.lines, query_start + mapped)) } } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs index 8a338529a2..6939744c45 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs @@ -1,7 +1,8 @@ //! Authored locations for native cast and set-operation findings that carry no span. -use crate::constants::{DATETIME_TYPE, TIMESTAMP_TYPE}; +use crate::constants::DATETIME_TYPE; use polyglot_sql::tokens::{Token, TokenType}; +use sqlbuild_core::constants::TIMESTAMP_TYPE; use crate::sql_tokens::main::projection_spans::projection_spans; use polyglot_sql::{Dialect, DialectType, ValidationError}; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/semantic_validation/main.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main.rs index cc9d6f58cf..6f02155b91 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -1,10 +1,10 @@ //! Stable SQLBuild boundary around Polyglot schema-aware validation. -pub(crate) mod diagnostics; -pub(crate) mod map_diagnostics; -pub(crate) mod normalize; -pub(crate) mod normalize_batch; -pub(crate) mod normalize_dialect; +pub mod diagnostics; +pub mod map_diagnostics; +pub mod normalize; +pub mod normalize_batch; +pub mod normalize_dialect; use crate::semantic_validation::models::ProjectCatalog; use polyglot_sql::{ @@ -49,14 +49,14 @@ fn default_dialect() -> String { "generic".to_string() } -pub(crate) fn validation_json(request_json: &str) -> Result { +pub fn validation_json(request_json: &str) -> Result { let request: ValidationRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; let result = validation_result(request)?; serde_json::to_string(&result).map_err(|error| error.to_string()) } -pub(crate) fn validations_json(request_json: &str) -> Result { +pub fn validations_json(request_json: &str) -> Result { let requests: Vec = serde_json::from_str(request_json).map_err(|error| error.to_string())?; if requests.is_empty() { diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs similarity index 88% rename from crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index 792630ca23..2efcff0313 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -1,6 +1,6 @@ use crate::semantic_validation::types::DiagnosticRow; -pub(crate) fn binding_diagnostics( +pub fn binding_diagnostics( sql: &str, dialect: &str, rows: Vec, diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/map_diagnostics.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main/map_diagnostics.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize.rs similarity index 72% rename from crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main/normalize.rs index add0711ac1..c248bf48d6 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize.rs @@ -1,6 +1,6 @@ use crate::semantic_validation::models::NormalizationInput; -pub(crate) fn normalize_analysis_sql(request: NormalizationInput) -> Result { +pub fn normalize_analysis_sql(request: NormalizationInput) -> Result { crate::semantic_validation::_helpers::normalization::normalize_analysis_sql( &request.sql, &request.dialect, diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_batch.rs similarity index 88% rename from crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main/normalize_batch.rs index 0ad8408745..947b2c7ff9 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_batch.rs @@ -1,11 +1,11 @@ use rayon::ThreadPool; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use crate::bindings::main::catch_compiler_panic::catch_compiler_panic; use crate::semantic_validation::types::NormalizationRequest; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; /// Normalize every request, keeping each failure in place so callers raise the first in order. -pub(crate) fn normalize_analysis_sqls( +pub fn normalize_analysis_sqls( dialect: &str, requests: Vec, pool: Option<&ThreadPool>, diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_dialect.rs similarity index 51% rename from crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs rename to crates/sqlbuild-analysis/src/semantic_validation/main/normalize_dialect.rs index f492ec101e..c47fca03ae 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/normalize_dialect.rs @@ -1,3 +1,3 @@ -pub(crate) fn normalize_dialect_sql(sql: &str, dialect: &str) -> Result { +pub fn normalize_dialect_sql(sql: &str, dialect: &str) -> Result { crate::semantic_validation::_helpers::normalization::normalize_dialect_sql(sql, dialect) } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/mod.rs b/crates/sqlbuild-analysis/src/semantic_validation/mod.rs new file mode 100644 index 0000000000..74f62c9e0a --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/mod.rs @@ -0,0 +1,7 @@ +pub mod _helpers; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/models.rs b/crates/sqlbuild-analysis/src/semantic_validation/models.rs similarity index 64% rename from crates/sqlbuild-rules-native/src/semantic_validation/models.rs rename to crates/sqlbuild-analysis/src/semantic_validation/models.rs index 8e94c6ec28..ca907c3877 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/models.rs @@ -1,44 +1,40 @@ -//! Compile-owned native catalog, mapping state and Python-boundary requests. +//! Compile-owned native catalog, mapping state and boundary requests. use crate::semantic_validation::types::{Expansion, PreparedCompactAnalysis, ProbeKey, Relations}; use polyglot_sql::validation::SchemaTable; use polyglot_sql::{DialectType, SchemaValidationOptions}; -use pyo3::{FromPyObject, pyclass}; use rayon::ThreadPool; use std::collections::HashMap; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; #[derive(Default, Debug)] -pub(crate) struct Columns(pub(crate) Vec<(String, Option)>); +pub struct Columns(pub Vec<(String, Option)>); -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct CatalogInput { - pub(crate) dialect: String, - pub(crate) quoted_ignore_case: bool, - pub(crate) known_functions: Vec, - pub(crate) known_types: Vec, - pub(crate) relations: Relations, +#[derive(Debug)] +pub struct CatalogInput { + pub dialect: String, + pub quoted_ignore_case: bool, + pub known_functions: Vec, + pub known_types: Vec, + pub relations: Relations, } -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct NormalizationInput { - pub(crate) sql: String, - pub(crate) dialect: String, - pub(crate) stubs: HashMap, - pub(crate) placeholders: HashMap, +#[derive(Debug)] +pub struct NormalizationInput { + pub sql: String, + pub dialect: String, + pub stubs: HashMap, + pub placeholders: HashMap, } -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct PositionInput { - pub(crate) authored: String, - pub(crate) query: String, - pub(crate) expanded: String, - pub(crate) cleaned: String, - pub(crate) passes: Vec>, +#[derive(Debug)] +pub struct PositionInput { + pub authored: String, + pub query: String, + pub expanded: String, + pub cleaned: String, + pub passes: Vec>, } /// Memoized call-site function-name probes, owned by the catalog that validates a compile. @@ -47,8 +43,7 @@ pub(crate) struct FunctionProbes { pub(crate) suggestions: Mutex>>, } -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct ProjectCatalog { +pub struct ProjectCatalog { pub(crate) dialect: DialectType, pub(crate) options: SchemaValidationOptions, pub(crate) quoted_ignore_case: bool, @@ -63,15 +58,13 @@ pub(crate) struct ProjectCatalog { } /// One resolved compact analysis batch that runs without borrowing its project catalog. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CompactAnalysisJob { +pub struct CompactAnalysisJob { pub(super) analysis: Mutex>, pub(super) catalog: ProjectCatalog, } -#[pyclass(module = "sqlbuild._native")] #[derive(Debug)] -pub(crate) struct BindingPositions { +pub struct BindingPositions { pub(super) authored: String, pub(super) query_start: Option, pub(super) lines: Vec, diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_function_names.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_normalize_batch.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_positions.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_positions.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_positions.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_positions.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_semantic_validation.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_semantic_validation.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_semantic_validation.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_semantic_validation.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_type_locations.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs rename to crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/types.rs b/crates/sqlbuild-analysis/src/semantic_validation/types.rs similarity index 63% rename from crates/sqlbuild-rules-native/src/semantic_validation/types.rs rename to crates/sqlbuild-analysis/src/semantic_validation/types.rs index 87ebd9353e..a6a8770c55 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/types.rs @@ -1,11 +1,11 @@ use crate::semantic_validation::models::{Columns, ProjectCatalog}; use std::collections::HashMap; -pub(crate) type Relations = HashMap; -pub(crate) type BindingRequest = (String, Vec<(String, bool)>, Relations); +pub type Relations = HashMap; +pub type BindingRequest = (String, Vec<(String, bool)>, Relations); /// One analysis SQL with its relation stubs and placeholder defaults. -pub(crate) type NormalizationRequest = (String, HashMap, HashMap); -pub(crate) type DiagnosticRow = ( +pub type NormalizationRequest = (String, HashMap, HashMap); +pub type DiagnosticRow = ( String, String, Option, @@ -14,7 +14,7 @@ pub(crate) type DiagnosticRow = ( Option, String, ); -pub(crate) type Expansion = (usize, usize, usize, usize); +pub type Expansion = (usize, usize, usize, usize); pub(crate) type ProbeKey = (polyglot_sql::DialectType, String, usize); /// Compact analysis whose catalog schemas are resolved; it runs with an options-only catalog. pub(crate) type PreparedCompactAnalysis = diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs rename to crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs index d1ca266e10..852563f311 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/builtin_functions.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs @@ -49,7 +49,7 @@ pub(crate) fn is_builtin_function(dialect: DialectType, upper_name: &str) -> boo } /// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] pub(crate) fn builtin_function_names(dialect: DialectType) -> Vec { let typed: &HashSet = match dialect { DialectType::DuckDB => &HashSet::new(), diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs new file mode 100644 index 0000000000..a2df8fb07e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub mod builtin_functions; +pub mod reserved_keywords; +pub mod token_context; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/_helpers/reserved_keywords.rs rename to crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/token_context.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/_helpers/token_context.rs rename to crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs similarity index 95% rename from crates/sqlbuild-rules-native/src/sql_tokens/constants.rs rename to crates/sqlbuild-analysis/src/sql_tokens/constants.rs index ebe02b15a0..6036eb9bf5 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -9,7 +9,7 @@ pub(crate) const CALL_OPENER: &str = "("; /// Field names after `.`, path keys after `:`, and parameter names after `@` or `$`. pub(crate) const NAME_PREFIXES: [&str; 4] = [".", ":", "@", "$"]; /// Built-in calls with keyword syntax inside their parentheses, which catalogues do not list. -pub(crate) const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ +pub const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ "CAST", "TRY_CAST", "SAFE_CAST", diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs new file mode 100644 index 0000000000..ec0c9e4260 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs @@ -0,0 +1,6 @@ +use polyglot_sql::DialectType; + +/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. +pub fn builtin_function_names(dialect: DialectType) -> Vec { + crate::sql_tokens::_helpers::builtin_functions::builtin_function_names(dialect) +} diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs index d921e2a80a..d1bfb837f9 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/canonical_tokens.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs @@ -8,7 +8,7 @@ use crate::sql_tokens::main::token_texts::token_texts; use crate::sql_tokens::models::CanonicalToken; /// Drop whitespace and comments from tokenized `sql` and upper-case case-insensitive tokens. -pub(crate) fn canonical_tokens( +pub fn canonical_tokens( sql: &str, tokens: &[Token], dialect: &Dialect, diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs similarity index 89% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs index 46645f9c08..fc29290477 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/case_folding.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs @@ -7,7 +7,7 @@ use crate::sql_tokens::_helpers::token_context::TokenContext; use crate::sql_tokens::main::name_positions::name_positions; /// Return, for every token, whether the formatter may recase it and the fingerprint folds it. -pub(crate) fn foldable_tokens(raws: &[String], tokens: &[Token], dialect: &Dialect) -> Vec { +pub fn foldable_tokens(raws: &[String], tokens: &[Token], dialect: &Dialect) -> Vec { let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); let names = name_positions(&uppers); let mut foldable = Vec::with_capacity(tokens.len()); diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs similarity index 89% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs index c1c40c63ad..6ea92ef370 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/is_unquoted_word.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs @@ -1,7 +1,7 @@ //! Unquoted SQL word recognition shared by canonical tokens and the formatter. /// Return whether `text` is an unquoted ASCII word whose letter case carries no meaning. -pub(crate) fn is_unquoted_word(text: &str) -> bool { +pub fn is_unquoted_word(text: &str) -> bool { let mut characters = text.chars(); characters .next() diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs new file mode 100644 index 0000000000..f073e8f1d7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs @@ -0,0 +1,9 @@ +#[cfg(any(test, feature = "test-support"))] +pub mod builtin_function_names; +pub mod canonical_tokens; +pub mod case_folding; +pub mod is_unquoted_word; +pub mod name_positions; +pub mod projection_spans; +pub mod query_fingerprint; +pub mod token_texts; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs index 91027cae60..b056a9bbef 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/name_positions.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs @@ -3,7 +3,7 @@ use crate::sql_tokens::constants::{ALIAS_KEYWORD, NAME_PREFIXES}; /// Return, for every upper-cased token, whether it follows `.`, `:`, `@`, `$` or `AS`. -pub(crate) fn name_positions(uppers: &[String]) -> Vec { +pub fn name_positions(uppers: &[String]) -> Vec { let mut positions = Vec::with_capacity(uppers.len()); let mut previous: Option<&str> = None; for upper in uppers { diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs similarity index 93% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs index 9a261b47b5..5965bd7875 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs @@ -5,7 +5,7 @@ use polyglot_sql::tokens::{Token, TokenType}; use crate::sql_tokens::constants::PROJECTION_ENDS; /// Spans of each top-level projection item of the select starting at token `select`. -pub(crate) fn projection_spans(tokens: &[Token], select: usize) -> Vec<(usize, usize)> { +pub fn projection_spans(tokens: &[Token], select: usize) -> Vec<(usize, usize)> { let mut spans: Vec<(usize, usize)> = Vec::new(); let mut depth: usize = 0; let mut start: Option = None; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs similarity index 91% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs index d162134f48..4cf7751adc 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs @@ -8,7 +8,7 @@ use sha2::{Digest, Sha256}; use crate::sql_tokens::main::canonical_tokens::canonical_tokens; /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. -pub(crate) fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { +pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { let dialect_type = DialectType::from_str(dialect_name) .map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?; let dialect = Dialect::get(dialect_type); diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs similarity index 85% rename from crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs rename to crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs index f0f29ada3a..8467369ccd 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/token_texts.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs @@ -3,7 +3,7 @@ use polyglot_sql::tokens::Token; /// Return the authored text of every token in `sql`. -pub(crate) fn token_texts(sql: &str, tokens: &[Token]) -> Result, String> { +pub fn token_texts(sql: &str, tokens: &[Token]) -> Result, String> { let characters: Vec = sql.chars().collect(); let mut texts = Vec::with_capacity(tokens.len()); for token in tokens { diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs similarity index 54% rename from crates/sqlbuild-rules-native/src/sql_tokens/mod.rs rename to crates/sqlbuild-analysis/src/sql_tokens/mod.rs index 4e8cc6fdd9..9904d54d66 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs @@ -1,9 +1,9 @@ //! SQLBuild's canonical SQL token stream: authored tokens without layout or comments. -pub(crate) mod _helpers; -pub(crate) mod constants; -pub(crate) mod main; -pub(crate) mod models; +pub mod _helpers; +pub mod constants; +pub mod main; +pub mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/models.rs b/crates/sqlbuild-analysis/src/sql_tokens/models.rs similarity index 77% rename from crates/sqlbuild-rules-native/src/sql_tokens/models.rs rename to crates/sqlbuild-analysis/src/sql_tokens/models.rs index c31021a7b3..b18ebb1a3a 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/models.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/models.rs @@ -2,6 +2,6 @@ /// One authored token with layout removed: keywords and built-in function names upper-cased. #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct CanonicalToken { - pub(crate) text: String, +pub struct CanonicalToken { + pub text: String, } diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/bigquery.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/bigquery.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/databricks.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/databricks.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/duckdb.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/generic.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/postgres.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/snowflake.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/tsql.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/reserved_keywords/tsql.txt rename to crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/tests.rs rename to crates/sqlbuild-analysis/src/sql_tokens/tests.rs diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/tests/test_canonical_tokens.rs rename to crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_tokens/tests/test_types.rs rename to crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml new file mode 100644 index 0000000000..c6657c84e0 --- /dev/null +++ b/crates/sqlbuild-core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "sqlbuild-core" +version.workspace = true +description = "Shared constants and the native panic boundary of SQLBuild." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs new file mode 100644 index 0000000000..423634b200 --- /dev/null +++ b/crates/sqlbuild-core/src/constants.rs @@ -0,0 +1,6 @@ +pub const SQL_WILDCARD: &str = "*"; +pub const TIMESTAMP_TYPE: &str = "TIMESTAMP"; +pub const OPEN_PAREN: &str = "("; +pub const CLOSE_PAREN: &str = ")"; +/// The error text every caught native panic carries across the boundary. +pub const PANIC_MESSAGE: &str = "NativeCompilerError: native SQL compilation panicked"; diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs new file mode 100644 index 0000000000..3912636b36 --- /dev/null +++ b/crates/sqlbuild-core/src/lib.rs @@ -0,0 +1,6 @@ +//! Shared foundations of the SQLBuild native workspace: constants and the panic boundary. + +#![forbid(unsafe_code)] + +pub mod constants; +pub mod panics; diff --git a/crates/sqlbuild-core/src/panics/main/catch_compiler_panic.rs b/crates/sqlbuild-core/src/panics/main/catch_compiler_panic.rs new file mode 100644 index 0000000000..bd356ea29e --- /dev/null +++ b/crates/sqlbuild-core/src/panics/main/catch_compiler_panic.rs @@ -0,0 +1,11 @@ +//! Convert unwinding native failures into ordinary named compiler errors. + +use crate::constants::PANIC_MESSAGE; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +pub fn catch_compiler_panic(operation: impl FnOnce() -> Result) -> Result { + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(result) => result, + Err(_) => Err(PANIC_MESSAGE.to_owned()), + } +} diff --git a/crates/sqlbuild-core/src/panics/main/is_compiler_panic.rs b/crates/sqlbuild-core/src/panics/main/is_compiler_panic.rs new file mode 100644 index 0000000000..51b2e937c2 --- /dev/null +++ b/crates/sqlbuild-core/src/panics/main/is_compiler_panic.rs @@ -0,0 +1,5 @@ +use crate::constants::PANIC_MESSAGE; + +pub fn is_compiler_panic(message: &str) -> bool { + message == PANIC_MESSAGE +} diff --git a/crates/sqlbuild-core/src/panics/main/mod.rs b/crates/sqlbuild-core/src/panics/main/mod.rs new file mode 100644 index 0000000000..c40fe7bcb0 --- /dev/null +++ b/crates/sqlbuild-core/src/panics/main/mod.rs @@ -0,0 +1,2 @@ +pub mod catch_compiler_panic; +pub mod is_compiler_panic; diff --git a/crates/sqlbuild-core/src/panics/mod.rs b/crates/sqlbuild-core/src/panics/mod.rs new file mode 100644 index 0000000000..25f92fd32f --- /dev/null +++ b/crates/sqlbuild-core/src/panics/mod.rs @@ -0,0 +1,3 @@ +pub mod main; +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/bindings/tests.rs b/crates/sqlbuild-core/src/panics/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/tests.rs rename to crates/sqlbuild-core/src/panics/tests.rs diff --git a/crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs b/crates/sqlbuild-core/src/panics/tests/test_panics.rs similarity index 83% rename from crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs rename to crates/sqlbuild-core/src/panics/tests/test_panics.rs index b25d9d08b1..f0992c1245 100644 --- a/crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs +++ b/crates/sqlbuild-core/src/panics/tests/test_panics.rs @@ -1,5 +1,6 @@ -use crate::bindings::_helpers::panics::{catch_compiler_panic, is_compiler_panic}; -use crate::bindings::tests::test_types::PanicTestCase; +use crate::panics::main::catch_compiler_panic::catch_compiler_panic; +use crate::panics::main::is_compiler_panic::is_compiler_panic; +use crate::panics::tests::test_types::PanicTestCase; #[test] fn given_native_failure_when_crossing_python_boundary_then_only_unwinds_become_named_compiler_errors() diff --git a/crates/sqlbuild-rules-native/src/bindings/tests/test_types.rs b/crates/sqlbuild-core/src/panics/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/tests/test_types.rs rename to crates/sqlbuild-core/src/panics/tests/test_types.rs diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml new file mode 100644 index 0000000000..14f33f0798 --- /dev/null +++ b/crates/sqlbuild-python/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "sqlbuild-python" +version.workspace = true +description = "The sqlbuild._native Python extension module." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "_native" +crate-type = ["cdylib", "rlib"] + +[features] +default = [] +extension-module = ["pyo3/extension-module"] + +[dependencies] +fensu-policy.workspace = true +mimalloc.workspace = true +pyo3.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-core.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-sqltext.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/allocator.rs b/crates/sqlbuild-python/src/bindings/_helpers/allocator.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/_helpers/allocator.rs rename to crates/sqlbuild-python/src/bindings/_helpers/allocator.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs new file mode 100644 index 0000000000..8adfad3d40 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/catalog.rs @@ -0,0 +1,124 @@ +//! Python methods of the compile-owned binding catalog and its compact analysis jobs. + +use crate::bindings::_helpers::conversions; +use crate::bindings::_helpers::normalization_results::normalization_results; +use crate::bindings::_helpers::panics::compiler_error; +use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; +use crate::bindings::types::CompilerDetach; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::PyBytes; +use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::types::{ + BindingRequest, DiagnosticRow, NormalizationRequest, Relations, +}; +use std::collections::HashMap; + +#[pymethods] +impl ProjectCatalog { + fn inferred_schema( + &self, + py: Python<'_>, + sql: &str, + #[pyo3(from_py_with = conversions::columns)] columns: validation::Columns, + #[pyo3(from_py_with = conversions::relations)] inputs: Relations, + ) -> PyResult>> { + py.compiler_detach(|| self.inner.inferred_schema(sql, columns, inputs)) + .map_err(compiler_error) + } + + #[new] + fn new(request: CatalogInput) -> PyResult { + validation::ProjectCatalog::new(request.into()) + .map(|inner| Self { inner }) + .map_err(PyValueError::new_err) + } + + fn update_relations( + &mut self, + #[pyo3(from_py_with = conversions::relations)] relations: Relations, + ) { + self.inner.update_relations(relations); + } + + fn with_relations( + &self, + #[pyo3(from_py_with = conversions::relations)] relations: Relations, + ) -> Self { + Self { + inner: self.inner.with_relations(relations), + } + } + + fn update_analysis( + &mut self, + #[pyo3(from_py_with = conversions::analysis_relations)] relations: HashMap< + String, + (validation::Columns, validation::Columns), + >, + ) { + self.inner.update_analysis(relations); + } + + fn register_override( + &mut self, + #[pyo3(from_py_with = conversions::relations)] relations: Relations, + ) -> usize { + self.inner.register_override(relations) + } + + fn validation_payload( + &self, + #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + ) -> PyResult { + self.inner + .validation_payload(requests) + .map_err(compiler_error) + } + + /// Resolve a compact batch now; the job runs later while the catalog keeps changing. + fn prepare_compact(&self, py: Python<'_>, payload: &[u8]) -> PyResult { + let text = std::str::from_utf8(payload) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.compiler_detach(|| self.inner.prepare_compact(text)) + .map(|inner| CompactAnalysisJob { inner }) + .map_err(compiler_error) + } + + fn binding_results( + &self, + py: Python<'_>, + #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + ) -> PyResult>> { + py.compiler_detach(|| self.inner.binding_results(requests)) + .map_err(compiler_error) + } + + /// Normalize a preparation batch on the idle analysis pool, else in turn, in one detached call. + fn normalize_analysis_sqls( + &self, + py: Python<'_>, + dialect: &str, + requests: Vec, + ) -> PyResult>> { + let results = py + .compiler_detach(|| self.inner.normalize_analysis_sqls(dialect, requests)) + .map_err(compiler_error)?; + normalization_results(py, results) + } +} + +#[pymethods] +impl CompactAnalysisJob { + fn run<'py>(&self, py: Python<'py>) -> PyResult> { + let analysis = self + .inner + .take_analysis() + .ok_or_else(|| PyValueError::new_err("compact analysis job already ran"))?; + let result = py + .compiler_detach(|| self.inner.run(analysis)) + .map_err(compiler_error)?; + Ok(PyBytes::new(py, result.as_bytes())) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/conversions.rs new file mode 100644 index 0000000000..fb5d9a8901 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/conversions.rs @@ -0,0 +1,72 @@ +//! Read catalog relation shapes from Python mappings in their authored column order. + +use pyo3::prelude::{Bound, FromPyObject, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::semantic_validation::models::Columns; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; +use std::collections::HashMap; + +/// One relation's columns, read from a `{name: type}` mapping. +struct ColumnList(Columns); + +/// Relation shapes keyed by relation name. +struct RelationMap(Relations); + +/// One binding request: SQL, its referenced relations and per-request relation overrides. +struct BindingRequestItem(BindingRequest); + +impl<'py> FromPyObject<'py> for ColumnList { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + columns(value).map(Self) + } +} + +impl<'py> FromPyObject<'py> for RelationMap { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + relations(value).map(Self) + } +} + +impl<'py> FromPyObject<'py> for BindingRequestItem { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + let (sql, references, overrides): (String, Vec<(String, bool)>, RelationMap) = + value.extract()?; + Ok(Self((sql, references, overrides.0))) + } +} + +pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult { + value + .downcast::()? + .iter() + .map(|(name, data_type)| Ok((name.extract()?, data_type.extract()?))) + .collect::>>() + .map(Columns) +} + +pub(crate) fn relations(value: &Bound<'_, PyAny>) -> PyResult { + let dict = value.cast::()?; + let mut relations = HashMap::with_capacity(dict.len()); + for (name, shape) in dict { + relations.insert(name.extract()?, columns(&shape)?); + } + Ok(relations) +} + +pub(crate) fn analysis_relations( + value: &Bound<'_, PyAny>, +) -> PyResult> { + let dict = value.cast::()?; + let mut relations = HashMap::with_capacity(dict.len()); + for (name, shapes) in dict { + let name: String = name.extract()?; + let (types, nullability): (ColumnList, ColumnList) = shapes.extract()?; + relations.insert(name, (types.0, nullability.0)); + } + Ok(relations) +} + +pub(crate) fn binding_requests(value: &Bound<'_, PyAny>) -> PyResult> { + let requests: Vec = value.extract()?; + Ok(requests.into_iter().map(|request| request.0).collect()) +} diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs similarity index 74% rename from crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs rename to crates/sqlbuild-python/src/bindings/_helpers/functions.rs index 267114a012..8bd0052338 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/functions.rs @@ -6,12 +6,12 @@ use pyo3::prelude::{ use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; -use crate::configuration::main::load; -use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; -use crate::engine::main::{evaluate, evaluate_parsed, parse_parts}; -use crate::models::CatalogueResponse; -use crate::models::ParsedRulesRequest; -use crate::rules::main::{catalogue, selected_codes}; +use crate::bindings::models::ParsedRulesRequest; +use sqlbuild_rules::configuration::main::load; +use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use sqlbuild_rules::engine::main::{evaluate, evaluate_parsed, parse_parts}; +use sqlbuild_rules::models::CatalogueResponse; +use sqlbuild_rules::rules::main::{catalogue, selected_codes}; use std::sync::Mutex; const SKILL_OWNER: &str = "sqlbuild"; @@ -22,10 +22,12 @@ const SKILL_IDENTITY: &str = "sqlbuild-rules"; #[pyfunction] fn normalize_analysis_sql( py: Python<'_>, - request: crate::semantic_validation::models::NormalizationInput, + request: crate::bindings::models::NormalizationInput, ) -> PyResult { py.compiler_detach(|| { - crate::semantic_validation::main::normalize::normalize_analysis_sql(request) + sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( + request.into(), + ) }) .map_err(value_error) } @@ -33,7 +35,9 @@ fn normalize_analysis_sql( #[pyfunction] fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { py.compiler_detach(|| { - crate::semantic_validation::main::normalize_dialect::normalize_dialect_sql(sql, dialect) + sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql( + sql, dialect, + ) }) .map_err(value_error) } @@ -43,16 +47,16 @@ fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, dialect: &str, - requests: Vec, + requests: Vec, ) -> PyResult>> { let results = py.compiler_detach(|| { Ok( - crate::semantic_validation::main::normalize_batch::normalize_analysis_sqls( + sqlbuild_analysis::semantic_validation::main::normalize_batch::normalize_analysis_sqls( dialect, requests, None, ), ) }); - crate::bindings::main::normalization_results::normalization_results( + crate::bindings::_helpers::normalization_results::normalization_results( py, results.map_err(value_error)?, ) @@ -63,10 +67,12 @@ fn binding_diagnostics( py: Python<'_>, sql: &str, dialect: &str, - rows: Vec, -) -> PyResult> { + rows: Vec, +) -> PyResult> { py.compiler_detach(|| { - crate::semantic_validation::main::diagnostics::binding_diagnostics(sql, dialect, rows) + sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( + sql, dialect, rows, + ) }) .map_err(value_error) } @@ -116,20 +122,24 @@ fn evaluate_parsed_rules( #[pyfunction] fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { - py.compiler_detach(|| crate::engine::main::custom_host::run_custom_host_json(spec_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_rules::engine::main::custom_host::run_custom_host_json(spec_json) + }) + .map_err(value_error) } #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::sql_lint::main::engine::lint_json(request_json)) + py.compiler_detach(|| sqlbuild_rules::sql_lint::main::engine::lint_json(request_json)) .map_err(value_error) } #[pyfunction] fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::engine::main::finalize::finalize_findings_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_rules::engine::main::finalize::finalize_findings_json(request_json) + }) + .map_err(value_error) } /// One SQL lint preparation request, read from a Python mapping. @@ -146,9 +156,9 @@ struct LintPreparationRequest { fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, -) -> PyResult> { +) -> PyResult> { py.compiler_detach(|| { - crate::sql_lint::main::preparation::prepare( + sqlbuild_rules::sql_lint::main::preparation::prepare( &request.expanded, &request.before_expansion, &request.prior_sites, @@ -163,13 +173,13 @@ fn prepare_lint_sql( fn prepare_lint_sql_batch( py: Python<'_>, requests: Vec, -) -> PyResult)>> { +) -> PyResult)>> { py.compiler_detach(|| { Ok(requests .iter() .map(|request| { - crate::bindings::_helpers::panics::catch_compiler_panic(|| { - crate::sql_lint::main::preparation::prepare( + sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( &request.expanded, &request.before_expansion, &request.prior_sites, @@ -186,70 +196,82 @@ fn prepare_lint_sql_batch( #[pyfunction] fn lint_backtick_identifiers(dialect: &str) -> PyResult { compiler_guard(|| { - Ok(crate::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect)) + Ok(sqlbuild_rules::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect)) }) } #[pyfunction] fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_rules::sql_lint::main::batch_engine::lint_batch_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::sql_lint::main::formatter::format_json(request_json)) + py.compiler_detach(|| sqlbuild_rules::sql_lint::main::formatter::format_json(request_json)) .map_err(value_error) } #[pyfunction] fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::sql_lint::main::batch_formatter::format_batch_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_rules::sql_lint::main::batch_formatter::format_batch_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { py.compiler_detach(|| { - crate::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) + sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect) }) .map_err(value_error) } #[pyfunction(name = "validate_sql_with_schema_json")] fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::semantic_validation::main::validation_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::validation_json(request_json) + }) + .map_err(value_error) } #[pyfunction(name = "validate_sql_with_schemas_json")] fn schema_validations_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::semantic_validation::main::validations_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::validations_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::semantic_usage::main::analyze_json(request_json)) + py.compiler_detach(|| sqlbuild_analysis::semantic_usage::main::analyze_json(request_json)) .map_err(value_error) } #[pyfunction] fn analyze_column_references_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::column_references::main::analyze::analyze_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_analysis::column_references::main::analyze::analyze_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::query_analysis::main::analyze::analyze_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_analysis::query_analysis::main::analyze::analyze_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::query_analysis::main::analyze_project::analyze_project_json(request_json) + sqlbuild_analysis::query_analysis::main::analyze_project::analyze_project_json(request_json) }) .map_err(value_error) } @@ -257,7 +279,7 @@ fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult< #[pyfunction] fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::query_analysis::main::analyze_project_compact::analyze_project_compact_json( + sqlbuild_analysis::query_analysis::main::analyze_project_compact::analyze_project_compact_json( request_json, ) }) @@ -266,14 +288,16 @@ fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> P #[pyfunction] fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| crate::compiler::main::sql_test_rendering::render_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_test_rendering::render_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn plan_and_render_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::compiler::main::sql_test_planning::plan_and_render_json(request_json) + sqlbuild_analysis::compiler::main::sql_test_planning::plan_and_render_json(request_json) }) .map_err(value_error) } @@ -281,7 +305,9 @@ fn plan_and_render_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResul #[pyfunction] fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::compiler::main::sql_test_chain_resolution::resolve_chains_json(request_json) + sqlbuild_analysis::compiler::main::sql_test_chain_resolution::resolve_chains_json( + request_json, + ) }) .map_err(value_error) } @@ -289,7 +315,7 @@ fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult< #[pyfunction] fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::compiler::main::sql_test_difference_sampling::render_difference_sample_json( + sqlbuild_analysis::compiler::main::sql_test_difference_sampling::render_difference_sample_json( request_json, ) }) @@ -299,16 +325,16 @@ fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> #[pyfunction] fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { - crate::compiler::main::sql_test_extraction::extract_batch_json(request_json) + sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json) }) .map_err(value_error) } fn authored_value_to_python( py: Python<'_>, - value: crate::compiler::models::AuthoredValue, + value: sqlbuild_sqltext::compiler::models::AuthoredValue, ) -> PyResult> { - use crate::compiler::models::AuthoredValue; + use sqlbuild_sqltext::compiler::models::AuthoredValue; match value { AuthoredValue::Null => Ok(py.None()), @@ -352,7 +378,7 @@ fn authored_value_to_python( fn map_to_python( py: Python<'_>, - values: Vec<(String, crate::compiler::models::AuthoredValue)>, + values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, ) -> PyResult> { let result = PyDict::new(py); for (key, value) in values { @@ -373,7 +399,7 @@ fn hook_marker( py: Python<'_>, kind: &str, name: String, - kwargs: Vec<(String, crate::compiler::models::AuthoredValue)>, + kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, ) -> PyResult> { let payload = PyTuple::new( py, @@ -387,7 +413,7 @@ fn hook_marker( fn optional_authored_value_to_python( py: Python<'_>, - value: Option, + value: Option, ) -> PyResult>> { value .map(|item| authored_value_to_python(py, item)) @@ -403,7 +429,9 @@ type ParsedModelHeader = ( #[pyfunction] fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { let parsed = py - .compiler_detach(|| crate::compiler::main::model_header_parsing::parse_batch(&headers)) + .compiler_detach(|| { + sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) + }) .map_err(value_error)?; parsed .into_iter() @@ -423,9 +451,7 @@ fn match_model_headers( contents: Vec, ) -> PyResult>> { py.compiler_detach(|| { - Ok(crate::compiler::main::model_header_matching::match_batch( - &contents, - )) + Ok(sqlbuild_sqltext::compiler::main::model_header_matching::match_batch(&contents)) }) .map_err(value_error) } @@ -433,7 +459,8 @@ fn match_model_headers( #[pyfunction] fn tokenize_model_header(header: &str) -> PyResult> { compiler_guard(|| { - crate::compiler::main::model_header_tokenizing::tokenize_one(header).map_err(value_error) + sqlbuild_sqltext::compiler::main::model_header_tokenizing::tokenize_one(header) + .map_err(value_error) }) } @@ -443,17 +470,23 @@ fn substitute_static_project_vars( variables: Vec<(String, String)>, ) -> PyResult)>> { compiler_guard(|| { - Ok(crate::compiler::main::sql_interpolation::substitute_batch( - &sqls, &variables, - )) + Ok( + sqlbuild_sqltext::compiler::main::sql_interpolation::substitute_batch( + &sqls, &variables, + ), + ) }) } #[pyfunction] fn extract_static_sql_references( sql: &str, -) -> PyResult>> { - compiler_guard(|| Ok(crate::compiler::main::sql_references::extract(sql))) +) -> PyResult>> { + compiler_guard(|| { + Ok(sqlbuild_sqltext::compiler::main::sql_references::extract( + sql, + )) + }) } #[pyfunction] @@ -520,9 +553,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { .py() .get_type::(), )?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs new file mode 100644 index 0000000000..83143c2dbf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -0,0 +1,7 @@ +mod allocator; +mod catalog; +pub(crate) mod conversions; +pub(crate) mod functions; +mod normalization_results; +pub(crate) mod panics; +mod positions; diff --git a/crates/sqlbuild-rules-native/src/bindings/main/normalization_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/main/normalization_results.rs rename to crates/sqlbuild-python/src/bindings/_helpers/normalization_results.rs diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs b/crates/sqlbuild-python/src/bindings/_helpers/panics.rs similarity index 72% rename from crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs rename to crates/sqlbuild-python/src/bindings/_helpers/panics.rs index 1c0a23fd9a..623dfb3b53 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/panics.rs @@ -3,9 +3,14 @@ use pyo3::Python; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::{PyErr, create_exception}; +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use crate::bindings::types::CompilerDetach; create_exception!(_native, NativeCompilerError, PyRuntimeError); -const PANIC_MESSAGE: &str = "NativeCompilerError: native SQL compilation panicked"; pub(crate) fn compiler_error(error: impl std::fmt::Display) -> PyErr { let message = error.to_string(); @@ -24,9 +29,6 @@ pub(crate) fn compiler_guard( Err(_) => Err(NativeCompilerError::new_err(PANIC_MESSAGE)), } } -use std::panic::{AssertUnwindSafe, catch_unwind}; - -use crate::bindings::types::CompilerDetach; impl CompilerDetach for Python<'_> { fn compiler_detach Result + Send>( @@ -36,16 +38,3 @@ impl CompilerDetach for Python<'_> { self.detach(|| catch_compiler_panic(operation)) } } - -pub(crate) fn catch_compiler_panic( - operation: impl FnOnce() -> Result, -) -> Result { - match catch_unwind(AssertUnwindSafe(operation)) { - Ok(result) => result, - Err(_) => Err(PANIC_MESSAGE.to_owned()), - } -} - -pub(crate) fn is_compiler_panic(message: &str) -> bool { - message == PANIC_MESSAGE -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/positions.rs b/crates/sqlbuild-python/src/bindings/_helpers/positions.rs new file mode 100644 index 0000000000..fc3da95977 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/positions.rs @@ -0,0 +1,33 @@ +//! Python methods of authored binding positions. + +use crate::bindings::_helpers::panics::compiler_guard; +use crate::bindings::models::{BindingPositions, PositionInput}; +use pyo3::exceptions::PyValueError; +use pyo3::{PyResult, pymethods}; +use sqlbuild_analysis::semantic_validation::models as validation; + +#[pymethods] +impl BindingPositions { + #[new] + fn new(request: PositionInput) -> PyResult { + compiler_guard(|| { + validation::BindingPositions::new(request.into()) + .map(|inner| Self { inner }) + .map_err(PyValueError::new_err) + }) + } + + fn position( + &self, + start: Option, + line: Option, + column: Option, + message: &str, + ) -> PyResult<(Option, Option)> { + compiler_guard(|| { + self.inner + .position(start, line, column, message) + .map_err(PyValueError::new_err) + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/main/mod.rs b/crates/sqlbuild-python/src/bindings/main/mod.rs new file mode 100644 index 0000000000..f862bee245 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/main/mod.rs @@ -0,0 +1 @@ +pub mod register; diff --git a/crates/sqlbuild-rules-native/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/main/register.rs rename to crates/sqlbuild-python/src/bindings/main/register.rs diff --git a/crates/sqlbuild-rules-native/src/bindings/mod.rs b/crates/sqlbuild-python/src/bindings/mod.rs similarity index 71% rename from crates/sqlbuild-rules-native/src/bindings/mod.rs rename to crates/sqlbuild-python/src/bindings/mod.rs index 5d64841a74..d887a75d22 100644 --- a/crates/sqlbuild-rules-native/src/bindings/mod.rs +++ b/crates/sqlbuild-python/src/bindings/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod _helpers; pub mod main; -#[cfg(test)] -mod tests; +pub(crate) mod models; pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs new file mode 100644 index 0000000000..26fc63d576 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -0,0 +1,98 @@ +//! Python-facing classes and mapping requests, converted to native models at the boundary. + +use pyo3::{FromPyObject, pyclass}; +use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::types::{Expansion, Relations}; +use sqlbuild_rules::models::EvaluateRequest; +use std::collections::HashMap; +use std::sync::Mutex; + +/// A decoded rules request held natively so the caller can release its encoded payloads. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub(crate) struct ParsedRulesRequest { + pub(crate) request: Mutex>, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct CatalogInput { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, + #[pyo3(from_py_with = crate::bindings::_helpers::conversions::relations)] + pub(crate) relations: Relations, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct NormalizationInput { + pub(crate) sql: String, + pub(crate) dialect: String, + pub(crate) stubs: HashMap, + pub(crate) placeholders: HashMap, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct PositionInput { + pub(crate) authored: String, + pub(crate) query: String, + pub(crate) expanded: String, + pub(crate) cleaned: String, + pub(crate) passes: Vec>, +} + +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub(crate) struct ProjectCatalog { + pub(crate) inner: validation::ProjectCatalog, +} + +/// One resolved compact analysis batch that runs without borrowing its project catalog. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CompactAnalysisJob { + pub(crate) inner: validation::CompactAnalysisJob, +} + +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub(crate) struct BindingPositions { + pub(crate) inner: validation::BindingPositions, +} + +impl From for validation::CatalogInput { + fn from(input: CatalogInput) -> Self { + Self { + dialect: input.dialect, + quoted_ignore_case: input.quoted_ignore_case, + known_functions: input.known_functions, + known_types: input.known_types, + relations: input.relations, + } + } +} + +impl From for validation::NormalizationInput { + fn from(input: NormalizationInput) -> Self { + Self { + sql: input.sql, + dialect: input.dialect, + stubs: input.stubs, + placeholders: input.placeholders, + } + } +} + +impl From for validation::PositionInput { + fn from(input: PositionInput) -> Self { + Self { + authored: input.authored, + query: input.query, + expanded: input.expanded, + cleaned: input.cleaned, + passes: input.passes, + } + } +} diff --git a/crates/sqlbuild-rules-native/src/bindings/types.rs b/crates/sqlbuild-python/src/bindings/types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/bindings/types.rs rename to crates/sqlbuild-python/src/bindings/types.rs diff --git a/crates/sqlbuild-python/src/lib.rs b/crates/sqlbuild-python/src/lib.rs new file mode 100644 index 0000000000..7c2526111e --- /dev/null +++ b/crates/sqlbuild-python/src/lib.rs @@ -0,0 +1,3 @@ +#![forbid(unsafe_code)] + +pub mod bindings; diff --git a/crates/sqlbuild-rules-native/build.rs b/crates/sqlbuild-rules-native/build.rs deleted file mode 100644 index a95c309a28..0000000000 --- a/crates/sqlbuild-rules-native/build.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Derive a content identity for this native build so local caches notice source changes. -//! -//! The identity hashes every file under the listed crate entries plus the workspace lockfile, -//! keyed by `/`-separated paths relative to the crate root so it matches across platforms. -//! Entries that do not exist (for example the lockfile outside a workspace checkout) are left -//! out of both the hash and the `rerun-if-changed` list, because Cargo reruns a build script on -//! every build when a watched path is missing. - -use sha2::{Digest, Sha256}; -use std::path::{Component, Path, PathBuf}; - -const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"]; -const WORKSPACE_LOCKFILE: &str = "../../Cargo.lock"; - -fn main() -> Result<(), String> { - let crate_dir = - PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?); - let mut files: Vec<(String, PathBuf)> = Vec::new(); - for entry in HASHED_ENTRIES { - let path = crate_dir.join(entry); - if path.exists() { - println!("cargo:rerun-if-changed={}", path.display()); - collect_files(&crate_dir, &path, &mut files)?; - } - } - files.sort(); - let lockfile = crate_dir.join(WORKSPACE_LOCKFILE); - if lockfile.is_file() { - println!("cargo:rerun-if-changed={}", lockfile.display()); - files.push((WORKSPACE_LOCKFILE.to_owned(), lockfile)); - } - let mut digest = Sha256::new(); - for (label, file) in &files { - digest.update(label.as_bytes()); - digest.update([0]); - digest.update(std::fs::read(file).map_err(|error| error.to_string())?); - digest.update([0]); - } - let identity = format!("{:x}", digest.finalize()); - println!( - "cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}", - &identity[..16] - ); - Ok(()) -} - -fn portable_label(crate_dir: &Path, path: &Path) -> Result { - let relative = path - .strip_prefix(crate_dir) - .map_err(|error| error.to_string())?; - let parts: Vec = relative - .components() - .map(|component| match component { - Component::Normal(part) => Ok(part.to_string_lossy().into_owned()), - other => Err(format!("unexpected path component {other:?}")), - }) - .collect::>()?; - Ok(parts.join("/")) -} - -fn collect_files( - crate_dir: &Path, - path: &Path, - files: &mut Vec<(String, PathBuf)>, -) -> Result<(), String> { - if path.is_file() { - files.push((portable_label(crate_dir, path)?, path.to_path_buf())); - return Ok(()); - } - if !path.is_dir() { - return Ok(()); - } - for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? { - collect_files( - crate_dir, - &entry.map_err(|error| error.to_string())?.path(), - files, - )?; - } - Ok(()) -} diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs deleted file mode 100644 index 6539c78531..0000000000 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod allocator; -pub(crate) mod functions; -pub(crate) mod panics; diff --git a/crates/sqlbuild-rules-native/src/bindings/main/catch_compiler_panic.rs b/crates/sqlbuild-rules-native/src/bindings/main/catch_compiler_panic.rs deleted file mode 100644 index f87fc5b8ec..0000000000 --- a/crates/sqlbuild-rules-native/src/bindings/main/catch_compiler_panic.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) fn catch_compiler_panic( - operation: impl FnOnce() -> Result, -) -> Result { - crate::bindings::_helpers::panics::catch_compiler_panic(operation) -} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs b/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs deleted file mode 100644 index bfc30b985b..0000000000 --- a/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn compiler_error(error: impl std::fmt::Display) -> pyo3::PyErr { - crate::bindings::_helpers::panics::compiler_error(error) -} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs b/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs deleted file mode 100644 index eff844d185..0000000000 --- a/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) fn compiler_guard( - operation: impl FnOnce() -> pyo3::PyResult, -) -> pyo3::PyResult { - crate::bindings::_helpers::panics::compiler_guard(operation) -} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/mod.rs b/crates/sqlbuild-rules-native/src/bindings/main/mod.rs deleted file mode 100644 index 9f6415af70..0000000000 --- a/crates/sqlbuild-rules-native/src/bindings/main/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) mod catch_compiler_panic; -pub(crate) mod compiler_error; -pub(crate) mod compiler_guard; -pub(crate) mod normalization_results; -pub mod register; diff --git a/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs b/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs deleted file mode 100644 index d1334fae40..0000000000 --- a/crates/sqlbuild-rules-native/src/column_references/main/analyze.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn analyze_json(request_json: &str) -> Result { - crate::column_references::references::analyze_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/column_references/main/mod.rs b/crates/sqlbuild-rules-native/src/column_references/main/mod.rs deleted file mode 100644 index 135b61fdc6..0000000000 --- a/crates/sqlbuild-rules-native/src/column_references/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod analyze; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/mod.rs deleted file mode 100644 index d4ab4655d5..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod model_headers; -pub(crate) mod sql_interpolation; -pub(crate) mod sql_references; -pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs deleted file mode 100644 index 2c80363b66..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod matching; -pub(crate) mod tokenization; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/mod.rs deleted file mode 100644 index 4d943c2ae2..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod substitution; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/mod.rs deleted file mode 100644 index a19c34f070..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod extraction; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs deleted file mode 100644 index 28677d435b..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub(crate) mod cte_rename; -pub(crate) mod cte_slices; -pub(crate) mod cte_sql; -pub(crate) mod expected_columns; -pub(crate) mod extraction; -pub(crate) mod helper_scope; -pub(crate) mod markers; -pub(crate) mod planning; -pub(crate) mod relation_markers; -pub(crate) mod rendering; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs deleted file mode 100644 index 4059a51c27..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub(crate) mod model_header_matching; -pub(crate) mod model_header_parsing; -pub(crate) mod model_header_tokenizing; -pub(crate) mod sql_interpolation; -pub(crate) mod sql_references; -pub(crate) mod sql_test_chain_resolution; -pub(crate) mod sql_test_difference_sampling; -pub(crate) mod sql_test_extraction; -pub(crate) mod sql_test_fixture_facts; -pub(crate) mod sql_test_planning; -pub(crate) mod sql_test_rendering; diff --git a/crates/sqlbuild-rules-native/src/compiler/mod.rs b/crates/sqlbuild-rules-native/src/compiler/mod.rs deleted file mode 100644 index a104008c09..0000000000 --- a/crates/sqlbuild-rules-native/src/compiler/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; -pub(crate) mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules-native/src/configuration/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/configuration/_helpers/mod.rs deleted file mode 100644 index 1f9445fcd8..0000000000 --- a/crates/sqlbuild-rules-native/src/configuration/_helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod loading; diff --git a/crates/sqlbuild-rules-native/src/configuration/main/mod.rs b/crates/sqlbuild-rules-native/src/configuration/main/mod.rs deleted file mode 100644 index 22abc37c06..0000000000 --- a/crates/sqlbuild-rules-native/src/configuration/main/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod load; -pub(crate) mod setting_help; -pub(crate) mod toml_string_array; -pub(crate) mod validate; diff --git a/crates/sqlbuild-rules-native/src/configuration/mod.rs b/crates/sqlbuild-rules-native/src/configuration/mod.rs deleted file mode 100644 index 7de28dc3ff..0000000000 --- a/crates/sqlbuild-rules-native/src/configuration/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs deleted file mode 100644 index d6c96b34b0..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod cache; -pub(crate) mod custom_host; -pub(crate) mod evaluation; diff --git a/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs b/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs deleted file mode 100644 index b59958c6dd..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/main/custom_host.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn run_custom_host_json(spec_json: &str) -> Result { - crate::engine::_helpers::custom_host::run_custom_host_json(spec_json) -} diff --git a/crates/sqlbuild-rules-native/src/engine/main/evaluate.rs b/crates/sqlbuild-rules-native/src/engine/main/evaluate.rs deleted file mode 100644 index b575e8399b..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/main/evaluate.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn evaluate_json(request_json: &str) -> Result { - crate::engine::_helpers::evaluation::evaluate_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/engine/main/finalize.rs b/crates/sqlbuild-rules-native/src/engine/main/finalize.rs deleted file mode 100644 index e28abf1c9c..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/main/finalize.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn finalize_findings_json(request_json: &str) -> Result { - crate::engine::_helpers::evaluation::finalize_findings_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/engine/main/mod.rs b/crates/sqlbuild-rules-native/src/engine/main/mod.rs deleted file mode 100644 index 097f97d0fc..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/main/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) mod custom_host; -pub(crate) mod evaluate; -pub(crate) mod evaluate_parsed; -pub(crate) mod finalize; -pub(crate) mod parse_parts; diff --git a/crates/sqlbuild-rules-native/src/engine/mod.rs b/crates/sqlbuild-rules-native/src/engine/mod.rs deleted file mode 100644 index eca16fd3de..0000000000 --- a/crates/sqlbuild-rules-native/src/engine/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules-native/src/lib.rs b/crates/sqlbuild-rules-native/src/lib.rs deleted file mode 100644 index 10c52999f9..0000000000 --- a/crates/sqlbuild-rules-native/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -#![forbid(unsafe_code)] - -pub mod bindings; -mod column_references; -mod compiler; -mod configuration; -mod constants; -mod engine; -mod models; -mod query_analysis; -mod rules; -mod scope_metadata; -mod semantic_usage; -mod semantic_validation; -mod sql_lint; -mod sql_quality; -mod sql_scan; -mod sql_tokens; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/analyze.rs deleted file mode 100644 index 51af080d5b..0000000000 --- a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn analyze_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project.rs deleted file mode 100644 index 838dd720e6..0000000000 --- a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn analyze_project_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_project_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_compact.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_compact.rs deleted file mode 100644 index 5b8f2a7da5..0000000000 --- a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_compact.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn analyze_project_compact_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_project_compact_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs deleted file mode 100644 index 196001e275..0000000000 --- a/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod analyze; -pub(crate) mod analyze_project; -pub(crate) mod analyze_project_compact; -pub(crate) mod prepare_project_catalog; diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/mod.rs deleted file mode 100644 index dcb6787ba2..0000000000 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub(crate) mod catalogue; -pub(crate) mod contract_name_types; -pub(crate) mod domain_layout; -pub(crate) mod dynamic_contracts; -pub(crate) mod evaluation; -pub(crate) mod explicit_output_types; -pub(crate) mod model_layers; -pub(crate) mod numeric_decisions; -pub(crate) mod sql_test_rules; -pub(crate) mod typed_contract_columns; diff --git a/crates/sqlbuild-rules-native/src/rules/main/mod.rs b/crates/sqlbuild-rules-native/src/rules/main/mod.rs deleted file mode 100644 index 605292c0ac..0000000000 --- a/crates/sqlbuild-rules-native/src/rules/main/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub(crate) mod assemble_catalogue; -pub(crate) mod catalogue; -pub(crate) mod evaluate; -pub(crate) mod evaluate_project; -pub(crate) mod fingerprint; -pub(crate) mod quote_policy; -pub(crate) mod resolve_threshold_overrides; -pub(crate) mod select; -pub(crate) mod selected_codes; -pub(crate) mod sql_test_coverage; diff --git a/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs b/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs deleted file mode 100644 index a33a47e140..0000000000 --- a/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs +++ /dev/null @@ -1,4 +0,0 @@ -/// Return the quoting policy rules use when scanning SQL for `dialect_name`. -pub(crate) fn quote_policy(dialect_name: &str) -> crate::sql_scan::models::QuotePolicy { - crate::rules::_helpers::evaluation::rules_quote_policy(dialect_name) -} diff --git a/crates/sqlbuild-rules-native/src/rules/main/selected_codes.rs b/crates/sqlbuild-rules-native/src/rules/main/selected_codes.rs deleted file mode 100644 index 3ed47fb79d..0000000000 --- a/crates/sqlbuild-rules-native/src/rules/main/selected_codes.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) fn selected_codes_json(request_json: &str) -> Result { - crate::rules::_helpers::catalogue::selected_codes_json(request_json) -} diff --git a/crates/sqlbuild-rules-native/src/rules/mod.rs b/crates/sqlbuild-rules-native/src/rules/mod.rs deleted file mode 100644 index 6d50510794..0000000000 --- a/crates/sqlbuild-rules-native/src/rules/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; -pub(crate) mod models; diff --git a/crates/sqlbuild-rules-native/src/scope_metadata/mod.rs b/crates/sqlbuild-rules-native/src/scope_metadata/mod.rs deleted file mode 100644 index cb4c63f603..0000000000 --- a/crates/sqlbuild-rules-native/src/scope_metadata/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod main; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs deleted file mode 100644 index 11d6c2854e..0000000000 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod catalog; -pub(crate) mod diagnostics; -pub(crate) mod function_names; -pub(crate) mod identifiers; -pub(crate) mod normalization; -pub(crate) mod positions; -pub(crate) mod type_locations; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs deleted file mode 100644 index faf043af1c..0000000000 --- a/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod main; -pub(crate) mod models; -pub(crate) mod types; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs deleted file mode 100644 index c180f8d4ac..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub(crate) mod additional; -pub(crate) mod engine; -pub(crate) mod formatter; -pub(crate) mod formatter_syntax; -pub(crate) mod inline_relations; -pub(crate) mod join_predicates; -pub(crate) mod preparation; -pub(crate) mod printed_layout; -pub(crate) mod terminal_shape; -pub(crate) mod token_layout; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs deleted file mode 100644 index f5c37dd789..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod backtick_identifiers; -pub(crate) mod batch_engine; -pub(crate) mod batch_formatter; -pub(crate) mod catalogue; -pub(crate) mod engine; -pub(crate) mod formatter; -pub(crate) mod preparation; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs deleted file mode 100644 index f58548d888..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod diagnostics; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs deleted file mode 100644 index efa92dc9fc..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod dialect; -pub(crate) mod dollar; -pub(crate) mod parens; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs deleted file mode 100644 index 2d272199c3..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod comment_end; -pub(crate) mod dialect_matching_paren; -pub(crate) mod dialect_non_code_ranges; -pub(crate) mod matching_paren; -pub(crate) mod non_code_end; -pub(crate) mod quote_end; -pub(crate) mod skip_whitespace; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/mod.rs deleted file mode 100644 index 2ffdf440f3..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod _helpers; -pub(crate) mod constants; -pub(crate) mod main; -pub(crate) mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs deleted file mode 100644 index 8e8d072e5b..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_tokens/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod builtin_functions; -pub(crate) mod reserved_keywords; -pub(crate) mod token_context; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs deleted file mode 100644 index 7690ea8d16..0000000000 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) mod canonical_tokens; -pub(crate) mod case_folding; -pub(crate) mod is_unquoted_word; -pub(crate) mod name_positions; -pub(crate) mod projection_spans; -pub(crate) mod query_fingerprint; -pub(crate) mod token_texts; diff --git a/crates/sqlbuild-rules-native/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml similarity index 63% rename from crates/sqlbuild-rules-native/Cargo.toml rename to crates/sqlbuild-rules/Cargo.toml index f390c5fd0b..29ac27d0b5 100644 --- a/crates/sqlbuild-rules-native/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -1,31 +1,24 @@ [package] -name = "sqlbuild-rules-native" -version = "0.136.0" +name = "sqlbuild-rules" +version.workspace = true +description = "Built-in rules, SQL lint, quality checks and formatting for SQLBuild." edition.workspace = true license.workspace = true publish.workspace = true rust-version.workspace = true -[lib] -name = "_native" -crate-type = ["cdylib", "rlib"] - -[features] -default = [] -extension-module = ["pyo3/extension-module"] - [dependencies] fensu-policy.workspace = true globset.workspace = true -mimalloc.workspace = true polyglot-sql.workspace = true -polyglot-sql-function-catalogs.workspace = true -pyo3.workspace = true rayon.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-core.workspace = true +sqlbuild-sqltext.workspace = true stacker.workspace = true sqlparser.workspace = true toml.workspace = true @@ -34,6 +27,7 @@ toml.workspace = true sha2.workspace = true [dev-dependencies] +sqlbuild-analysis = { workspace = true, features = ["test-support"] } tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-rules/build.rs b/crates/sqlbuild-rules/build.rs new file mode 100644 index 0000000000..441748c703 --- /dev/null +++ b/crates/sqlbuild-rules/build.rs @@ -0,0 +1,108 @@ +//! Derive a content identity for this native build so local caches notice source changes. +//! +//! The identity hashes every file under the listed entries of every workspace crate plus the +//! workspace manifest (dependency features and build profiles) and lockfile, keyed by +//! `/`-separated paths relative to the crates directory so it matches across platforms. Entries +//! that do not exist (for example the lockfile outside a workspace checkout) are left out of both +//! the hash and the `rerun-if-changed` list, because Cargo reruns a build script on every build +//! when a watched path is missing. The hashed workspace files are exported so tests can assert +//! they are covered. + +use sha2::{Digest, Sha256}; +use std::path::{Component, Path, PathBuf}; + +const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"]; +const WORKSPACE_FILES: [&str; 2] = ["../../Cargo.toml", "../../Cargo.lock"]; + +fn main() -> Result<(), String> { + let crate_dir = + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?); + let crates_dir = crate_dir + .parent() + .ok_or("native crate has no parent directory")? + .to_path_buf(); + let mut files: Vec<(String, PathBuf)> = Vec::new(); + for workspace_crate in workspace_crates(&crates_dir)? { + for entry in HASHED_ENTRIES { + let path = workspace_crate.join(entry); + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + collect_files(&crates_dir, &path, &mut files)?; + } + } + } + files.sort(); + let mut workspace_files = Vec::new(); + for label in WORKSPACE_FILES { + let path = crate_dir.join(label); + if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + files.push((label.to_owned(), path)); + workspace_files.push(label); + } + } + let mut digest = Sha256::new(); + for (label, file) in &files { + digest.update(label.as_bytes()); + digest.update([0]); + digest.update(std::fs::read(file).map_err(|error| error.to_string())?); + digest.update([0]); + } + let identity = format!("{:x}", digest.finalize()); + println!( + "cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}", + &identity[..16] + ); + println!( + "cargo:rustc-env=SQLBUILD_NATIVE_HASHED_WORKSPACE_FILES={}", + workspace_files.join(",") + ); + Ok(()) +} + +/// Every sibling crate directory, so the identity covers the whole native extension. +fn workspace_crates(crates_dir: &Path) -> Result, String> { + let mut crates = Vec::new(); + for entry in std::fs::read_dir(crates_dir).map_err(|error| error.to_string())? { + let path = entry.map_err(|error| error.to_string())?.path(); + if path.join("Cargo.toml").is_file() { + crates.push(path); + } + } + crates.sort(); + Ok(crates) +} + +fn portable_label(root: &Path, path: &Path) -> Result { + let relative = path.strip_prefix(root).map_err(|error| error.to_string())?; + let parts: Vec = relative + .components() + .map(|component| match component { + Component::Normal(part) => Ok(part.to_string_lossy().into_owned()), + other => Err(format!("unexpected path component {other:?}")), + }) + .collect::>()?; + Ok(parts.join("/")) +} + +fn collect_files( + root: &Path, + path: &Path, + files: &mut Vec<(String, PathBuf)>, +) -> Result<(), String> { + if path.is_file() { + files.push((portable_label(root, path)?, path.to_path_buf())); + return Ok(()); + } + if !path.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? { + collect_files( + root, + &entry.map_err(|error| error.to_string())?.path(), + files, + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs rename to crates/sqlbuild-rules/src/configuration/_helpers/loading.rs diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs new file mode 100644 index 0000000000..f33faa4b7b --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs @@ -0,0 +1 @@ +pub mod loading; diff --git a/crates/sqlbuild-rules-native/src/configuration/main/load.rs b/crates/sqlbuild-rules/src/configuration/main/load.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/configuration/main/load.rs rename to crates/sqlbuild-rules/src/configuration/main/load.rs index b82f3f527a..43ca577d63 100644 --- a/crates/sqlbuild-rules-native/src/configuration/main/load.rs +++ b/crates/sqlbuild-rules/src/configuration/main/load.rs @@ -1,5 +1,5 @@ use std::path::Path; -pub(crate) fn load_config_json(project_dir: &Path) -> Result { +pub fn load_config_json(project_dir: &Path) -> Result { crate::configuration::_helpers::loading::load_config_json(project_dir) } diff --git a/crates/sqlbuild-rules/src/configuration/main/mod.rs b/crates/sqlbuild-rules/src/configuration/main/mod.rs new file mode 100644 index 0000000000..fbd112cce3 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/main/mod.rs @@ -0,0 +1,4 @@ +pub mod load; +pub mod setting_help; +pub mod toml_string_array; +pub mod validate; diff --git a/crates/sqlbuild-rules-native/src/configuration/main/setting_help.rs b/crates/sqlbuild-rules/src/configuration/main/setting_help.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/configuration/main/setting_help.rs rename to crates/sqlbuild-rules/src/configuration/main/setting_help.rs diff --git a/crates/sqlbuild-rules-native/src/configuration/main/toml_string_array.rs b/crates/sqlbuild-rules/src/configuration/main/toml_string_array.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/configuration/main/toml_string_array.rs rename to crates/sqlbuild-rules/src/configuration/main/toml_string_array.rs diff --git a/crates/sqlbuild-rules-native/src/configuration/main/validate.rs b/crates/sqlbuild-rules/src/configuration/main/validate.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/configuration/main/validate.rs rename to crates/sqlbuild-rules/src/configuration/main/validate.rs diff --git a/crates/sqlbuild-rules/src/configuration/mod.rs b/crates/sqlbuild-rules/src/configuration/mod.rs new file mode 100644 index 0000000000..0b794b2977 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/mod.rs @@ -0,0 +1,2 @@ +pub mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs similarity index 54% rename from crates/sqlbuild-rules-native/src/constants.rs rename to crates/sqlbuild-rules/src/constants.rs index 6e4aec2a8e..d9527daa60 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -1,5 +1,5 @@ -pub(crate) const API_VERSION: u32 = 1; -pub(crate) const NATIVE_BUILD_IDENTITY: &str = concat!( +pub const API_VERSION: u32 = 1; +pub const NATIVE_BUILD_IDENTITY: &str = concat!( env!("CARGO_PKG_VERSION"), "+", env!("SQLBUILD_NATIVE_SOURCE_HASH") @@ -19,39 +19,15 @@ pub(crate) const STAGING_LAYER_DIRECTORY: &str = "staging"; pub(crate) const INTERMEDIATE_LAYER_DIRECTORY: &str = "intermediate"; pub(crate) const MART_LAYER_DIRECTORY: &str = "mart"; pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; -pub(crate) const DBT_REFERENCE_KIND: &str = "dbt_ref"; -pub(crate) const TABLE_FUNCTION_REFERENCE_KIND: &str = "table_function"; pub(crate) const VIEW_MATERIALIZATION: &str = "view"; pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; pub(crate) const SELECT_STAR_MODEL_RULE_CODE: &str = "SQBRMODEL102"; pub(crate) const BOOLEAN_TYPE: &str = "BOOLEAN"; -pub(crate) const TIMESTAMP_TYPE: &str = "TIMESTAMP"; -pub(crate) const DATETIME_TYPE: &str = "DATETIME"; pub(crate) const DATE_TYPE: &str = "DATE"; pub(crate) const NEGATION_OPERATOR: &str = "-"; pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; -pub(crate) const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; -pub(crate) const SQL_WILDCARD: &str = "*"; -pub(crate) const UNKNOWN_SQL_TYPE: &str = "UNKNOWN"; -pub(crate) const VARCHAR_SQL_TYPE: &str = "VARCHAR"; -pub(crate) const COLUMNS_KEY: &str = "columns"; -pub(crate) const CONSTANT_CALL: &str = "constant"; -pub(crate) const INLINE_SQL_HOOK: &str = "inline_sql"; -pub(crate) const MACRO_TEST_MODE: &str = "macro"; -pub(crate) const OPEN_PAREN: &str = "("; -pub(crate) const CLOSE_PAREN: &str = ")"; pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; -pub(crate) const OUTSIDE_KEY: &str = "outside"; -pub(crate) const SQL_HOOK: &str = "sql"; -pub(crate) const TABLE_FUNCTION_TEST_MODE: &str = "table_fn"; -pub(crate) const THRESHOLDS_KEY: &str = "thresholds"; -pub(crate) const TYPE_KEY: &str = "type"; -pub(crate) const UDF_TEST_MODE: &str = "udf"; -pub(crate) const SQL_TEST_ACTUAL_CTE: &str = "__actual"; -pub(crate) const SQL_TEST_EXPECTED_CTE: &str = "__expected"; -pub(crate) const SQL_TEST_ACTUAL_CTE_PREFIX: &str = "__actual__"; -pub(crate) const QUOTED_IDENTIFIER_DELIMITER_BYTES: usize = 2; pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/_helpers/cache.rs rename to crates/sqlbuild-rules/src/engine/_helpers/cache.rs diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs b/crates/sqlbuild-rules/src/engine/_helpers/custom_host.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/_helpers/custom_host.rs rename to crates/sqlbuild-rules/src/engine/_helpers/custom_host.rs diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs rename to crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs new file mode 100644 index 0000000000..ed7464e6d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub mod cache; +pub mod custom_host; +pub mod evaluation; diff --git a/crates/sqlbuild-rules/src/engine/main/custom_host.rs b/crates/sqlbuild-rules/src/engine/main/custom_host.rs new file mode 100644 index 0000000000..db33f6f8fc --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/custom_host.rs @@ -0,0 +1,3 @@ +pub fn run_custom_host_json(spec_json: &str) -> Result { + crate::engine::_helpers::custom_host::run_custom_host_json(spec_json) +} diff --git a/crates/sqlbuild-rules/src/engine/main/evaluate.rs b/crates/sqlbuild-rules/src/engine/main/evaluate.rs new file mode 100644 index 0000000000..9b038e4ae2 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/evaluate.rs @@ -0,0 +1,3 @@ +pub fn evaluate_json(request_json: &str) -> Result { + crate::engine::_helpers::evaluation::evaluate_json(request_json) +} diff --git a/crates/sqlbuild-rules-native/src/engine/main/evaluate_parsed.rs b/crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/engine/main/evaluate_parsed.rs rename to crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs index 3331c2a9be..81cc27c0fd 100644 --- a/crates/sqlbuild-rules-native/src/engine/main/evaluate_parsed.rs +++ b/crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs @@ -1,5 +1,5 @@ use crate::models::EvaluateRequest; -pub(crate) fn evaluate_parsed(request: EvaluateRequest) -> Result { +pub fn evaluate_parsed(request: EvaluateRequest) -> Result { crate::engine::_helpers::evaluation::evaluate_request(request) } diff --git a/crates/sqlbuild-rules/src/engine/main/finalize.rs b/crates/sqlbuild-rules/src/engine/main/finalize.rs new file mode 100644 index 0000000000..477f1980e8 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/finalize.rs @@ -0,0 +1,3 @@ +pub fn finalize_findings_json(request_json: &str) -> Result { + crate::engine::_helpers::evaluation::finalize_findings_json(request_json) +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs new file mode 100644 index 0000000000..d5349e927b --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -0,0 +1,5 @@ +pub mod custom_host; +pub mod evaluate; +pub mod evaluate_parsed; +pub mod finalize; +pub mod parse_parts; diff --git a/crates/sqlbuild-rules-native/src/engine/main/parse_parts.rs b/crates/sqlbuild-rules/src/engine/main/parse_parts.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/engine/main/parse_parts.rs rename to crates/sqlbuild-rules/src/engine/main/parse_parts.rs index 017234272a..6b11c2b7b6 100644 --- a/crates/sqlbuild-rules-native/src/engine/main/parse_parts.rs +++ b/crates/sqlbuild-rules/src/engine/main/parse_parts.rs @@ -1,6 +1,6 @@ use crate::models::EvaluateRequest; -pub(crate) fn parse_parts( +pub fn parse_parts( request_json: &[u8], model_jsons: &[&[u8]], model_digests: &[String], diff --git a/crates/sqlbuild-rules/src/engine/mod.rs b/crates/sqlbuild-rules/src/engine/mod.rs new file mode 100644 index 0000000000..d5e5e9f39a --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/mod.rs @@ -0,0 +1,5 @@ +pub mod _helpers; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests.rs rename to crates/sqlbuild-rules/src/engine/tests.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/helpers.rs rename to crates/sqlbuild-rules/src/engine/tests/helpers.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_domain_layout.rs b/crates/sqlbuild-rules/src/engine/tests/test_domain_layout.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/test_domain_layout.rs rename to crates/sqlbuild-rules/src/engine/tests/test_domain_layout.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs b/crates/sqlbuild-rules/src/engine/tests/test_empty_input_tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs rename to crates/sqlbuild-rules/src/engine/tests/test_empty_input_tests.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs rename to crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index ded80cec80..80dfd7d965 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -1363,3 +1363,29 @@ fn given_native_build_when_reading_cache_identity_then_version_carries_source_ha ); } } + +#[test] +fn given_workspace_checkout_when_deriving_build_identity_then_workspace_files_are_hashed() { + let test_cases = [ + test_types::HashedWorkspaceFileTestCase { + description: "dependency features and build profiles change the identity", + workspace_file: "../../Cargo.toml", + expected_hashed: true, + }, + test_types::HashedWorkspaceFileTestCase { + description: "resolved dependency versions change the identity", + workspace_file: "../../Cargo.lock", + expected_hashed: true, + }, + ]; + for test_case in &test_cases { + let hashed = env!("SQLBUILD_NATIVE_HASHED_WORKSPACE_FILES") + .split(',') + .any(|file| file == test_case.workspace_file); + assert_eq!( + hashed, test_case.expected_hashed, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/test_incremental_rules.rs rename to crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_scope_facts.rs b/crates/sqlbuild-rules/src/engine/tests/test_scope_facts.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/test_scope_facts.rs rename to crates/sqlbuild-rules/src/engine/tests/test_scope_facts.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs similarity index 97% rename from crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs rename to crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index afbd58d470..52a9a2f15f 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -1,13 +1,14 @@ -use crate::compiler::_helpers::sql_references::extraction::extract; use crate::engine::tests::test_types::{ DollarQuoteSiteTestCase, LintDialectSiteTestCase, SqlScannerTestCase, }; -use crate::rules::_helpers::evaluation::{normalize_rules_sql, rules_quote_policy}; +use crate::rules::_helpers::evaluation::normalize_rules_sql; use crate::rules::_helpers::numeric_decisions::compact_sql; use crate::sql_lint::_helpers::preparation::prepare; -use crate::sql_scan::main::matching_paren::matching_paren; -use crate::sql_scan::models::QuotePolicy; -use crate::sql_scan::models::Unclosed; +use sqlbuild_sqltext::compiler::main::sql_references::extract; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; +use sqlbuild_sqltext::sql_scan::models::Unclosed; #[test] fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scanner_is_characterised() diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs rename to crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs similarity index 97% rename from crates/sqlbuild-rules-native/src/engine/tests/test_types.rs rename to crates/sqlbuild-rules/src/engine/tests/test_types.rs index 0d348f2d96..8dbb4535ad 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::sql_scan::models::Unclosed; +use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(crate) struct NativeEvaluationTestCase { pub(crate) description: &'static str, @@ -112,6 +112,12 @@ pub(crate) struct BuildIdentityTestCase { pub(crate) expected_hash_length: usize, } +pub(crate) struct HashedWorkspaceFileTestCase { + pub(crate) description: &'static str, + pub(crate) workspace_file: &'static str, + pub(crate) expected_hashed: bool, +} + pub(crate) struct ThresholdFingerprintTestCase { pub(crate) description: &'static str, pub(crate) base_config: Value, diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs new file mode 100644 index 0000000000..ec95267231 --- /dev/null +++ b/crates/sqlbuild-rules/src/lib.rs @@ -0,0 +1,12 @@ +//! Built-in rules, SQL lint, quality checks and formatting, with their configuration. + +#![forbid(unsafe_code)] + +pub mod configuration; +pub mod constants; +pub mod engine; +pub mod models; +pub mod rules; +pub mod scope_metadata; +pub mod sql_lint; +pub mod sql_quality; diff --git a/crates/sqlbuild-rules-native/src/models.rs b/crates/sqlbuild-rules/src/models.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/models.rs rename to crates/sqlbuild-rules/src/models.rs index ac7197deac..b9bfbd3af2 100644 --- a/crates/sqlbuild-rules-native/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -1,8 +1,6 @@ -use pyo3::pyclass; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; -use std::sync::Mutex; use crate::constants::{ API_VERSION, BUILT_IN_RULE_NAMESPACE, CUSTOM_RULE_NAMESPACE, RULE_CODE_NUMBER_LENGTH, @@ -49,7 +47,7 @@ impl RulesCodeGrammar { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct CacheConfig { +pub struct CacheConfig { pub enabled: bool, } @@ -61,7 +59,7 @@ impl CacheConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct RuleException { +pub struct RuleException { pub rule: String, pub path: String, pub reason: String, @@ -69,7 +67,7 @@ pub(crate) struct RuleException { #[derive(Clone, Debug, Deserialize, Serialize, Eq, Ord, PartialEq, PartialOrd)] #[serde(deny_unknown_fields)] -pub(crate) struct GraphEdgeException { +pub struct GraphEdgeException { pub consumer: String, pub dependency: String, pub reason: String, @@ -77,7 +75,7 @@ pub(crate) struct GraphEdgeException { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct RuleIgnore { +pub struct RuleIgnore { pub rules: Vec, #[serde(default)] pub paths: Vec, @@ -88,14 +86,14 @@ pub(crate) struct RuleIgnore { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SelectStarAllow { +pub struct SelectStarAllow { pub paths: Vec, pub reason: String, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ThresholdOverride { +pub struct ThresholdOverride { pub paths: Vec, pub thresholds: BTreeMap, pub reason: String, @@ -103,7 +101,7 @@ pub(crate) struct ThresholdOverride { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct SqlTestRulesConfig { +pub struct SqlTestRulesConfig { pub pipeline_directory: String, } @@ -117,7 +115,7 @@ impl Default for SqlTestRulesConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct LayoutConfig { +pub struct LayoutConfig { pub levels: Vec, pub domain_roots: Vec, } @@ -138,7 +136,7 @@ impl Default for LayoutConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct RulesConfig { +pub struct RulesConfig { pub select: Vec, pub ignore: Vec, pub thresholds: BTreeMap, @@ -209,7 +207,7 @@ impl RulesConfig { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct Reference { +pub struct Reference { pub ref_kind: String, pub ref_name: String, pub ref_package: Option, @@ -217,7 +215,7 @@ pub(crate) struct Reference { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct Column { +pub struct Column { pub name: String, #[serde(rename = "type", alias = "data_type")] pub data_type: String, @@ -228,7 +226,7 @@ pub(crate) struct Column { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct DynamicColumnFamily { +pub struct DynamicColumnFamily { pub name: String, pub pivot_column: String, pub value_column: String, @@ -241,14 +239,14 @@ pub(crate) struct DynamicColumnFamily { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct DeclarationMember { +pub struct DeclarationMember { pub name: String, pub value: Value, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct Declaration { +pub struct Declaration { pub name: String, pub relative_path: String, pub members: Vec, @@ -259,7 +257,7 @@ pub(crate) struct Declaration { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum ResourceKind { +pub enum ResourceKind { Model, Test, Scenario, @@ -270,7 +268,7 @@ pub(crate) enum ResourceKind { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum DeclarationKind { +pub enum DeclarationKind { Macro, Enum, Constant, @@ -283,7 +281,7 @@ pub(crate) enum DeclarationKind { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum ScopeKind { +pub enum ScopeKind { Global, Inherited, Local, @@ -292,7 +290,7 @@ pub(crate) enum ScopeKind { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum UsageKind { +pub enum UsageKind { Runtime, Generated, DeclarationDependency, @@ -300,14 +298,14 @@ pub(crate) enum UsageKind { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum GrantKind { +pub enum GrantKind { ExpectedModel, TestedMacro, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum VisibilityReason { +pub enum VisibilityReason { Global, InheritedAncestor, LocalOwner, @@ -318,7 +316,7 @@ pub(crate) enum VisibilityReason { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum InaccessibleReason { +pub enum InaccessibleReason { LocalBoundary, SiblingScope, DescendantScope, @@ -329,13 +327,13 @@ pub(crate) enum InaccessibleReason { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum DiagnosticSeverity { +pub enum DiagnosticSeverity { Error, Warning, } #[derive(Clone, Debug, Deserialize, Serialize)] -pub(crate) enum ScopeDiagnosticCode { +pub enum ScopeDiagnosticCode { S001, S002, S003, @@ -364,14 +362,14 @@ pub(crate) enum ScopeDiagnosticCode { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeOwnershipRoot { +pub struct ScopeOwnershipRoot { pub path: String, pub resource_kind: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeResource { +pub struct ScopeResource { pub identity: String, pub kind: ResourceKind, pub name: String, @@ -382,7 +380,7 @@ pub(crate) struct ScopeResource { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeMacroMetadata { +pub struct ScopeMacroMetadata { pub parameters: Vec, pub dependencies: Vec, pub source_digest: String, @@ -390,20 +388,20 @@ pub(crate) struct ScopeMacroMetadata { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeEnumMember { +pub struct ScopeEnumMember { pub name: String, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeEnumMetadata { +pub struct ScopeEnumMetadata { pub members: Vec, pub scalar_type: String, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeConstantMetadata { +pub struct ScopeConstantMetadata { pub logical_type: String, pub collection_kind: Option, pub item_count: Option, @@ -413,7 +411,7 @@ pub(crate) struct ScopeConstantMetadata { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct ScopeDeclarationMetadata { +pub struct ScopeDeclarationMetadata { pub r#macro: Option, pub r#enum: Option, pub constant: Option, @@ -421,7 +419,7 @@ pub(crate) struct ScopeDeclarationMetadata { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeDeclaration { +pub struct ScopeDeclaration { pub identity: String, pub kind: DeclarationKind, pub name: String, @@ -445,7 +443,7 @@ pub(crate) struct ScopeDeclaration { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeUsage { +pub struct ScopeUsage { pub consumer: String, pub declaration: String, pub kind: UsageKind, @@ -455,7 +453,7 @@ pub(crate) struct ScopeUsage { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeGrant { +pub struct ScopeGrant { pub resource: String, pub declaration: String, pub through: String, @@ -464,7 +462,7 @@ pub(crate) struct ScopeGrant { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeVisibility { +pub struct ScopeVisibility { pub resource: String, pub declaration: String, pub reason: VisibilityReason, @@ -473,7 +471,7 @@ pub(crate) struct ScopeVisibility { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeInaccessible { +pub struct ScopeInaccessible { pub resource: String, pub declaration: String, pub reason: InaccessibleReason, @@ -481,7 +479,7 @@ pub(crate) struct ScopeInaccessible { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeDiagnostic { +pub struct ScopeDiagnostic { pub code: ScopeDiagnosticCode, pub message: String, pub severity: DiagnosticSeverity, @@ -494,7 +492,7 @@ pub(crate) struct ScopeDiagnostic { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeCompleteness { +pub struct ScopeCompleteness { pub discovery: bool, pub static_visibility: bool, pub runtime_usage: bool, @@ -505,7 +503,7 @@ pub(crate) struct ScopeCompleteness { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct ScopeIndexFacts { +pub struct ScopeIndexFacts { #[serde(deserialize_with = "crate::scope_metadata::main::deserialize_scope_schema_version")] pub schema_version: u32, pub ownership_roots: Vec, @@ -547,7 +545,7 @@ impl Default for ScopeIndexFacts { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct Model { +pub struct Model { #[serde(default)] pub sql_analysis_disabled: bool, pub name: String, @@ -575,7 +573,7 @@ pub(crate) struct Model { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum SqlTestMode { +pub enum SqlTestMode { Model, Macro, Udf, @@ -584,7 +582,7 @@ pub(crate) enum SqlTestMode { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum DirectTestResourceKind { +pub enum DirectTestResourceKind { Macro, Udf, TableFn, @@ -592,14 +590,14 @@ pub(crate) enum DirectTestResourceKind { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct TestedResource { +pub struct TestedResource { pub kind: DirectTestResourceKind, pub name: String, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SqlTestParameterFact { +pub struct SqlTestParameterFact { pub name: String, #[serde(rename = "type")] pub value_type: String, @@ -608,7 +606,7 @@ pub(crate) struct SqlTestParameterFact { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SqlTestParameterValueFact { +pub struct SqlTestParameterValueFact { pub name: String, #[serde(rename = "type")] pub value_type: String, @@ -617,7 +615,7 @@ pub(crate) struct SqlTestParameterValueFact { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SqlTestFact { +pub struct SqlTestFact { pub source_path: String, pub ownership_root: String, pub block_index: u64, @@ -657,14 +655,14 @@ pub(crate) struct SqlTestFact { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SqlTestCteFact { +pub struct SqlTestCteFact { pub name: String, pub sql: String, } #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct SqlScenarioFact { +pub struct SqlScenarioFact { pub source_path: String, pub ownership_root: String, pub name: String, @@ -677,7 +675,7 @@ pub(crate) struct SqlScenarioFact { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub(crate) struct CustomRule { +pub struct CustomRule { pub code: String, pub family: String, pub slug: String, @@ -704,7 +702,7 @@ pub(crate) struct CustomRule { #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(deny_unknown_fields)] -pub(crate) struct Fault { +pub struct Fault { #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub unevaluated: bool, pub code: String, @@ -717,7 +715,7 @@ pub(crate) struct Fault { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] -pub(crate) struct EvaluateRequest { +pub struct EvaluateRequest { pub version: u32, pub project_dir: String, pub dialect: String, @@ -751,13 +749,6 @@ pub(crate) struct ResolveRulesRequest { pub custom_rules: Vec, } -/// A decoded rules request held natively so the caller can release its encoded payloads. -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct ParsedRulesRequest { - pub(crate) request: Mutex>, -} - impl Default for EvaluateRequest { fn default() -> Self { Self { @@ -801,7 +792,7 @@ pub(crate) struct EvaluateResponse { } #[derive(Clone, Debug, Serialize)] -pub(crate) struct RuleGuidance { +pub struct RuleGuidance { pub good_example: String, pub anti_tautology: String, pub mutation_check: String, @@ -819,7 +810,7 @@ impl RuleGuidance { /// Inputs that can change a rule's findings; only `Model` findings are cached per model. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum RuleScope { +pub enum RuleScope { Model, Project, ModelAndProject, @@ -837,7 +828,7 @@ impl RuleScope { } #[derive(Clone, Debug, Serialize)] -pub(crate) struct RuleMetadata { +pub struct RuleMetadata { pub code: String, pub family: String, pub slug: String, @@ -868,7 +859,7 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } #[derive(Debug, Serialize)] -pub(crate) struct CatalogueResponse { +pub struct CatalogueResponse { pub version: u32, pub rules: Vec, } diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs rename to crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs index f505f71681..2579ae0d84 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/contract_name_types.rs @@ -1,5 +1,6 @@ -use crate::constants::{BOOLEAN_TYPE, DATE_TYPE, TIMESTAMP_TYPE}; +use crate::constants::{BOOLEAN_TYPE, DATE_TYPE}; use crate::models::{Column, RulesConfig}; +use sqlbuild_core::constants::TIMESTAMP_TYPE; const BOOLEAN_COLUMN_RULE: &str = "SQBRCONTRACT102"; const TIMESTAMP_COLUMN_RULE: &str = "SQBRCONTRACT103"; diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs b/crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs rename to crates/sqlbuild-rules/src/rules/_helpers/domain_layout.rs diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs b/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs rename to crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs similarity index 97% rename from crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs rename to crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index f4bf74d34a..e27f63926b 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -13,18 +13,17 @@ use crate::rules::_helpers::{ use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, }; -use crate::sql_scan::main::matching_paren::matching_paren; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::models::QuotePolicy; use globset::{Glob, GlobSetBuilder}; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; +use sqlbuild_sqltext::sql_scan::main::rules_dialect::rules_dialect; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; use sqlparser::ast::{ BinaryOperator, Expr, GroupByExpr, JoinConstraint, JoinOperator, Query, Select, SelectItem, SetExpr, Spanned, Statement, TableFactor, Value, Visit, Visitor, }; -use sqlparser::dialect::{ - BigQueryDialect, ClickHouseDialect, DatabricksDialect, Dialect, DuckDbDialect, GenericDialect, - MsSqlDialect, PostgreSqlDialect, SnowflakeDialect, -}; +use sqlparser::dialect::{Dialect, GenericDialect}; use sqlparser::parser::{Parser, ParserError, ParserOptions}; use sqlparser::tokenizer::{Token, Tokenizer}; use std::collections::{BTreeMap, BTreeSet, HashSet}; @@ -338,31 +337,6 @@ fn parse_with_dialect(sql: &str, dialect: &dyn Dialect) -> Result parser.parse_statements() } -pub(crate) fn rules_quote_policy(dialect_name: &str) -> QuotePolicy { - let dialect = rules_dialect(dialect_name); - let backslash_escapes = dialect.supports_string_literal_backslash_escape(); - QuotePolicy { - backtick_identifiers: dialect.is_delimited_identifier_start('`'), - single_quote_backslash_escapes: backslash_escapes, - double_quote_backslash_escapes: backslash_escapes - && !dialect.is_delimited_identifier_start('"'), - dollar_quotes: false, - } -} - -fn rules_dialect(name: &str) -> Box { - match name.to_ascii_lowercase().as_str() { - "bigquery" => Box::new(BigQueryDialect {}), - "clickhouse" => Box::new(ClickHouseDialect {}), - "databricks" => Box::new(DatabricksDialect {}), - "duckdb" => Box::new(DuckDbDialect {}), - "postgres" | "postgresql" => Box::new(PostgreSqlDialect {}), - "mssql" | "sqlserver" | "tsql" => Box::new(MsSqlDialect {}), - "snowflake" => Box::new(SnowflakeDialect {}), - _ => Box::new(GenericDialect {}), - } -} - pub(crate) fn normalize_rules_sql(dialect: &str, sql: &str) -> String { let policy = rules_quote_policy(dialect); let sql = normalize_table_function_calls(sql, policy); diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs rename to crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs diff --git a/crates/sqlbuild-rules/src/rules/_helpers/mod.rs b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs new file mode 100644 index 0000000000..18833e9d3d --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub mod catalogue; +pub mod contract_name_types; +pub mod domain_layout; +pub mod dynamic_contracts; +pub mod evaluation; +pub mod explicit_output_types; +pub mod model_layers; +pub mod numeric_decisions; +pub mod sql_test_rules; +pub mod typed_contract_columns; diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs rename to crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/numeric_decisions.rs b/crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs similarity index 91% rename from crates/sqlbuild-rules-native/src/rules/_helpers/numeric_decisions.rs rename to crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs index 3d861d3fb3..8f0522b65d 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/numeric_decisions.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/numeric_decisions.rs @@ -1,6 +1,6 @@ -use crate::sql_scan::main::comment_end::comment_end; -use crate::sql_scan::main::quote_end::quote_end; -use crate::sql_scan::models::QuotePolicy; +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::quote_end::quote_end; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; const CANONICAL_NUMERIC_DECISIONS: [&str; 3] = ["-1", "0", "1"]; diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs rename to crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 6743a60084..4c9fe1d5ed 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -1,4 +1,3 @@ -use crate::compiler::main::sql_test_fixture_facts::fixture_facts; use crate::models::{ DeclarationKind, EvaluateRequest, Fault, Model, ResourceKind, RuleMetadata, RulesConfig, ScopeResource, SqlScenarioFact, SqlTestCteFact, SqlTestFact, SqlTestMode, @@ -8,6 +7,7 @@ use crate::rules::_helpers::evaluation::{ unwrap_nested, }; use crate::rules::models::ProjectEvaluationRequest; +use sqlbuild_analysis::compiler::main::sql_test_fixture_facts::fixture_facts; use sqlparser::ast::{ BinaryOperator, Expr, FunctionArg, FunctionArgExpr, LimitClause, Query, Select, SetExpr, Statement, TableFactor, Value, Visit, Visitor, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs b/crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs rename to crates/sqlbuild-rules/src/rules/_helpers/typed_contract_columns.rs diff --git a/crates/sqlbuild-rules-native/src/rules/main/assemble_catalogue.rs b/crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/assemble_catalogue.rs rename to crates/sqlbuild-rules/src/rules/main/assemble_catalogue.rs diff --git a/crates/sqlbuild-rules-native/src/rules/main/catalogue.rs b/crates/sqlbuild-rules/src/rules/main/catalogue.rs similarity index 63% rename from crates/sqlbuild-rules-native/src/rules/main/catalogue.rs rename to crates/sqlbuild-rules/src/rules/main/catalogue.rs index ea0a4dc773..88c4b1aeac 100644 --- a/crates/sqlbuild-rules-native/src/rules/main/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/main/catalogue.rs @@ -1,5 +1,5 @@ use crate::models::RuleMetadata; -pub(crate) fn catalogue() -> Vec { +pub fn catalogue() -> Vec { crate::rules::_helpers::catalogue::catalogue() } diff --git a/crates/sqlbuild-rules-native/src/rules/main/evaluate.rs b/crates/sqlbuild-rules/src/rules/main/evaluate.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/evaluate.rs rename to crates/sqlbuild-rules/src/rules/main/evaluate.rs diff --git a/crates/sqlbuild-rules-native/src/rules/main/evaluate_project.rs b/crates/sqlbuild-rules/src/rules/main/evaluate_project.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/evaluate_project.rs rename to crates/sqlbuild-rules/src/rules/main/evaluate_project.rs diff --git a/crates/sqlbuild-rules-native/src/rules/main/fingerprint.rs b/crates/sqlbuild-rules/src/rules/main/fingerprint.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/fingerprint.rs rename to crates/sqlbuild-rules/src/rules/main/fingerprint.rs diff --git a/crates/sqlbuild-rules/src/rules/main/mod.rs b/crates/sqlbuild-rules/src/rules/main/mod.rs new file mode 100644 index 0000000000..bde8969bbb --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/mod.rs @@ -0,0 +1,9 @@ +pub mod assemble_catalogue; +pub mod catalogue; +pub mod evaluate; +pub mod evaluate_project; +pub mod fingerprint; +pub mod resolve_threshold_overrides; +pub mod select; +pub mod selected_codes; +pub mod sql_test_coverage; diff --git a/crates/sqlbuild-rules-native/src/rules/main/resolve_threshold_overrides.rs b/crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/resolve_threshold_overrides.rs rename to crates/sqlbuild-rules/src/rules/main/resolve_threshold_overrides.rs diff --git a/crates/sqlbuild-rules-native/src/rules/main/select.rs b/crates/sqlbuild-rules/src/rules/main/select.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/select.rs rename to crates/sqlbuild-rules/src/rules/main/select.rs diff --git a/crates/sqlbuild-rules/src/rules/main/selected_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs new file mode 100644 index 0000000000..d2c4ea82e5 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/main/selected_codes.rs @@ -0,0 +1,3 @@ +pub fn selected_codes_json(request_json: &str) -> Result { + crate::rules::_helpers::catalogue::selected_codes_json(request_json) +} diff --git a/crates/sqlbuild-rules-native/src/rules/main/sql_test_coverage.rs b/crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/main/sql_test_coverage.rs rename to crates/sqlbuild-rules/src/rules/main/sql_test_coverage.rs diff --git a/crates/sqlbuild-rules/src/rules/mod.rs b/crates/sqlbuild-rules/src/rules/mod.rs new file mode 100644 index 0000000000..f5f7fd5d12 --- /dev/null +++ b/crates/sqlbuild-rules/src/rules/mod.rs @@ -0,0 +1,3 @@ +pub mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-rules-native/src/rules/models.rs b/crates/sqlbuild-rules/src/rules/models.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/rules/models.rs rename to crates/sqlbuild-rules/src/rules/models.rs diff --git a/crates/sqlbuild-rules-native/src/scope_metadata/main.rs b/crates/sqlbuild-rules/src/scope_metadata/main.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/scope_metadata/main.rs rename to crates/sqlbuild-rules/src/scope_metadata/main.rs diff --git a/crates/sqlbuild-rules/src/scope_metadata/mod.rs b/crates/sqlbuild-rules/src/scope_metadata/mod.rs new file mode 100644 index 0000000000..2a043412b4 --- /dev/null +++ b/crates/sqlbuild-rules/src/scope_metadata/mod.rs @@ -0,0 +1 @@ +pub mod main; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/additional.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index 3d6b769bcb..caa570c1b4 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -2,16 +2,14 @@ use std::collections::{HashMap, HashSet}; use polyglot_sql::tokens::{Span, Token, TokenType}; -use crate::constants::{ - CLOSE_PAREN, EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT, OPEN_PAREN, - SQL_WILDCARD, -}; +use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; use crate::sql_lint::_helpers::engine::{ direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, significant_before, token_depths, }; use crate::sql_lint::constants::RELATION_MODIFIER_KEYWORDS; use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts}; +use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; const COUNT_ONE_LITERAL: &str = "1"; const QUALIFIED_REFERENCE_LENGTH: usize = 3; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter_syntax.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/formatter_syntax.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/inline_relations.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/inline_relations.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/join_predicates.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/join_predicates.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs new file mode 100644 index 0000000000..4b14cd8cb8 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub mod additional; +pub mod engine; +pub mod formatter; +pub mod formatter_syntax; +pub mod inline_relations; +pub mod join_predicates; +pub mod preparation; +pub mod printed_layout; +pub mod terminal_shape; +pub mod token_layout; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs similarity index 95% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index 6d520c10c5..1ad5815f42 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -5,11 +5,11 @@ use std::sync::LazyLock; use regex::Regex; -use crate::rules::main::quote_policy::quote_policy; use crate::sql_lint::types::{InterpolationSite, PreparedSql}; -use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; -use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::models::QuotePolicy; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; static SITE_START: LazyLock> = LazyLock::new(|| { Regex::new(r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()") diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/printed_layout.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/printed_layout.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/terminal_shape.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/terminal_shape.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs similarity index 98% rename from crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs rename to crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs index 83be59d0bd..eac207cbb8 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/token_layout.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs @@ -6,11 +6,11 @@ use polyglot_sql::Dialect; use polyglot_sql::tokens::{Token, TokenType}; use crate::sql_lint::constants::{CLOSE_PARENTHESIS, OPEN_PARENTHESIS}; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -use crate::sql_tokens::main::case_folding::foldable_tokens; -use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; -use crate::sql_tokens::main::name_positions::name_positions; -use crate::sql_tokens::main::token_texts::token_texts; +use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; +use sqlbuild_analysis::sql_tokens::main::case_folding::foldable_tokens; +use sqlbuild_analysis::sql_tokens::main::is_unquoted_word::is_unquoted_word; +use sqlbuild_analysis::sql_tokens::main::name_positions::name_positions; +use sqlbuild_analysis::sql_tokens::main::token_texts::token_texts; const TOKEN_PRESERVATION_FAILURE: &str = "native formatter would change authored SQL tokens, not only layout"; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/constants.rs b/crates/sqlbuild-rules/src/sql_lint/constants.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/constants.rs rename to crates/sqlbuild-rules/src/sql_lint/constants.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs b/crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs similarity index 72% rename from crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs rename to crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs index f0cc7a2a75..79670356f6 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs @@ -1,4 +1,4 @@ /// Return whether SQL lint treats backticks as identifier quotes for `dialect`. -pub(crate) fn backtick_identifiers(dialect: &str) -> bool { +pub fn backtick_identifiers(dialect: &str) -> bool { crate::sql_lint::_helpers::preparation::backtick_identifiers(dialect) } diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/batch_engine.rs b/crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/sql_lint/main/batch_engine.rs rename to crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs index aea9fd8ae8..2d7de066ba 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/batch_engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs @@ -2,7 +2,7 @@ use crate::sql_lint::_helpers::engine::lint; use crate::sql_lint::models::{LintRequest, LintResponse}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -pub(crate) fn lint_batch_json(request_json: &str) -> Result { +pub fn lint_batch_json(request_json: &str) -> Result { let requests: Vec = serde_json::from_str(request_json).map_err(|error| error.to_string())?; if requests.is_empty() { diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/batch_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/main/batch_formatter.rs similarity index 57% rename from crates/sqlbuild-rules-native/src/sql_lint/main/batch_formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/main/batch_formatter.rs index d278cd430e..d3920372af 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/batch_formatter.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/batch_formatter.rs @@ -1,5 +1,5 @@ use crate::sql_lint::_helpers::formatter::format_batch_json_impl; -pub(crate) fn format_batch_json(request_json: &str) -> Result { +pub fn format_batch_json(request_json: &str) -> Result { format_batch_json_impl(request_json) } diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/catalogue.rs b/crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/main/catalogue.rs rename to crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/engine.rs b/crates/sqlbuild-rules/src/sql_lint/main/engine.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/sql_lint/main/engine.rs rename to crates/sqlbuild-rules/src/sql_lint/main/engine.rs index 3572bb20c1..2533f4f949 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/engine.rs @@ -1,5 +1,5 @@ use crate::sql_lint::_helpers::engine::lint_json_impl; -pub(crate) fn lint_json(request_json: &str) -> Result { +pub fn lint_json(request_json: &str) -> Result { lint_json_impl(request_json) } diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/formatter.rs b/crates/sqlbuild-rules/src/sql_lint/main/formatter.rs similarity index 56% rename from crates/sqlbuild-rules-native/src/sql_lint/main/formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/main/formatter.rs index b3f810a67b..42da9c9c03 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/formatter.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/formatter.rs @@ -1,5 +1,5 @@ use crate::sql_lint::_helpers::formatter::format_json_impl; -pub(crate) fn format_json(request_json: &str) -> Result { +pub fn format_json(request_json: &str) -> Result { format_json_impl(request_json) } diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs new file mode 100644 index 0000000000..4a2daec8b0 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -0,0 +1,7 @@ +pub mod backtick_identifiers; +pub mod batch_engine; +pub mod batch_formatter; +pub mod catalogue; +pub mod engine; +pub mod formatter; +pub mod preparation; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/main/preparation.rs similarity index 94% rename from crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs rename to crates/sqlbuild-rules/src/sql_lint/main/preparation.rs index a1b2710ffd..71987caa78 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/preparation.rs @@ -2,7 +2,7 @@ use crate::sql_lint::types::PreparedSql; -pub(crate) fn prepare( +pub fn prepare( expanded: &str, before_expansion: &str, prior_sites: &[usize], diff --git a/crates/sqlbuild-rules-native/src/sql_lint/mod.rs b/crates/sqlbuild-rules/src/sql_lint/mod.rs similarity index 55% rename from crates/sqlbuild-rules-native/src/sql_lint/mod.rs rename to crates/sqlbuild-rules/src/sql_lint/mod.rs index e82420248a..1b51a640b0 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/mod.rs @@ -1,10 +1,10 @@ //! SQLBuild-owned generic SQL linting over the external Polyglot parser. -pub(crate) mod _helpers; +pub mod _helpers; mod constants; -pub(crate) mod main; -pub(crate) mod models; -pub(crate) mod types; +pub mod main; +pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/models.rs rename to crates/sqlbuild-rules/src/sql_lint/models.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests.rs rename to crates/sqlbuild-rules/src/sql_lint/tests.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md b/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md rename to crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/format_corpus.json b/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/fixtures/format_corpus.json rename to crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs index 43e3886f44..57aabf1ed2 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs @@ -6,10 +6,10 @@ use serde_json::{Value, json}; use crate::sql_lint::main::batch_formatter::format_batch_json; use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; -use crate::sql_tokens::_helpers::builtin_functions::builtin_function_names; -use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -use crate::sql_tokens::main::query_fingerprint::query_fingerprint; +use sqlbuild_analysis::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; +use sqlbuild_analysis::sql_tokens::main::builtin_function_names::builtin_function_names; +use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; +use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_authored_tokens.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_batch_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_batch_lint.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_comment_formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_empty_fixture_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_empty_fixture_lint.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_format_corpus.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_function_call_spacing.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_function_call_spacing.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs index 191db93296..b5d068a929 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs @@ -2,7 +2,7 @@ use serde_json::Value; use crate::sql_lint::tests::helpers::format_at_width; use crate::sql_lint::tests::test_types; -use crate::sql_tokens::main::query_fingerprint::query_fingerprint; +use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; #[test] fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently() diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_line_wrap.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_line_wrap.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_set_operation_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_set_operation_formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_types.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_unused_table_alias.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_unused_table_alias.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_lint/tests/test_values_formatter.rs rename to crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs diff --git a/crates/sqlbuild-rules-native/src/sql_lint/types.rs b/crates/sqlbuild-rules/src/sql_lint/types.rs similarity index 51% rename from crates/sqlbuild-rules-native/src/sql_lint/types.rs rename to crates/sqlbuild-rules/src/sql_lint/types.rs index 682c369422..1cb260e2c3 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/types.rs @@ -1,2 +1,2 @@ pub(crate) type InterpolationSite = (String, usize, usize, usize, usize, String); -pub(crate) type PreparedSql = (String, Vec, Vec); +pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs index 37232ab85c..8b5e6b2b3c 100644 --- a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs @@ -18,7 +18,7 @@ use crate::sql_quality::syntax::{ Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier, sources, }; -use crate::sql_tokens::main::projection_spans::projection_spans; +use sqlbuild_analysis::sql_tokens::main::projection_spans::projection_spans; const HARNESS_PREFIX: &str = "__"; const COLUMNS_FUNCTION: &str = "COLUMNS"; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/json_parses.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/keys.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/keys.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/literals.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/literals.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/ranking.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/ranking.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/removal_safety.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/syntax.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/_helpers/syntax.rs rename to crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/constants.rs b/crates/sqlbuild-rules/src/sql_quality/constants.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/constants.rs rename to crates/sqlbuild-rules/src/sql_quality/constants.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/main/diagnostics.rs b/crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/main/diagnostics.rs rename to crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs diff --git a/crates/sqlbuild-rules/src/sql_quality/main/mod.rs b/crates/sqlbuild-rules/src/sql_quality/main/mod.rs new file mode 100644 index 0000000000..4c630487f1 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_quality/main/mod.rs @@ -0,0 +1 @@ +pub mod diagnostics; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/mod.rs b/crates/sqlbuild-rules/src/sql_quality/mod.rs similarity index 83% rename from crates/sqlbuild-rules-native/src/sql_quality/mod.rs rename to crates/sqlbuild-rules/src/sql_quality/mod.rs index 20988413ce..520f7e2f3c 100644 --- a/crates/sqlbuild-rules-native/src/sql_quality/mod.rs +++ b/crates/sqlbuild-rules/src/sql_quality/mod.rs @@ -1,6 +1,6 @@ //! Enforceable SQL quality Rules with project-wide limits and ranking determinism proofs. -pub(crate) mod constants; +pub mod constants; #[path = "_helpers/cte_columns.rs"] mod cte_columns; #[path = "_helpers/json_parses.rs"] @@ -9,15 +9,15 @@ mod json_parses; mod keys; #[path = "_helpers/literals.rs"] mod literals; -pub(crate) mod main; -pub(crate) mod models; +pub mod main; +pub mod models; #[path = "_helpers/ranking.rs"] mod ranking; #[path = "_helpers/removal_safety.rs"] mod removal_safety; #[path = "_helpers/syntax.rs"] mod syntax; -pub(crate) mod types; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/models.rs b/crates/sqlbuild-rules/src/sql_quality/models.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/models.rs rename to crates/sqlbuild-rules/src/sql_quality/models.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests.rs b/crates/sqlbuild-rules/src/sql_quality/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests.rs rename to crates/sqlbuild-rules/src/sql_quality/tests.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests/helpers.rs rename to crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests/test_json_parses.rs rename to crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_rules.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_rules.rs rename to crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_scale.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_scale.rs rename to crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/tests/test_types.rs rename to crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs diff --git a/crates/sqlbuild-rules-native/src/sql_quality/types.rs b/crates/sqlbuild-rules/src/sql_quality/types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_quality/types.rs rename to crates/sqlbuild-rules/src/sql_quality/types.rs diff --git a/crates/sqlbuild-sqltext/Cargo.toml b/crates/sqlbuild-sqltext/Cargo.toml new file mode 100644 index 0000000000..09864fdb60 --- /dev/null +++ b/crates/sqlbuild-sqltext/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sqlbuild-sqltext" +version.workspace = true +description = "Lexical SQL scanning, model headers and static SQL text operations for SQLBuild." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +rayon.workspace = true +serde.workspace = true +sqlbuild-core.workspace = true +sqlparser.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs new file mode 100644 index 0000000000..efcfa91e8b --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub mod model_headers; +pub mod sql_interpolation; +pub mod sql_references; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs rename to crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/matching.rs diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs new file mode 100644 index 0000000000..8ca7357664 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs @@ -0,0 +1,2 @@ +pub mod matching; +pub mod tokenization; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/tokenization.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs similarity index 99% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/tokenization.rs rename to crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs index 7f0aa041c1..f9a2f8bfa5 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/tokenization.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/tokenization.rs @@ -5,9 +5,9 @@ use rayon::{ThreadPool, ThreadPoolBuilder}; use crate::compiler::models::AuthoredValue; use crate::constants::{ - CLOSE_PAREN, COLUMNS_KEY, CONSTANT_CALL, INLINE_SQL_HOOK, OPEN_PAREN, OUTSIDE_KEY, SQL_HOOK, - THRESHOLDS_KEY, TYPE_KEY, + COLUMNS_KEY, CONSTANT_CALL, INLINE_SQL_HOOK, OUTSIDE_KEY, SQL_HOOK, THRESHOLDS_KEY, TYPE_KEY, }; +use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN}; pub(crate) const MAX_TOKENIZER_WORKERS: usize = 4; pub(crate) const TOKENIZER_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/mod.rs new file mode 100644 index 0000000000..e43f6a7562 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/mod.rs @@ -0,0 +1 @@ +pub mod substitution; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_interpolation/substitution.rs rename to crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/extraction.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs similarity index 97% rename from crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/extraction.rs rename to crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs index ac05233798..1ab6ca4e89 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_references/extraction.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/extraction.rs @@ -1,5 +1,6 @@ //! Conservative native fast path for logical SQL reference extraction. +use crate::compiler::types::StaticReference; use crate::constants::{DBT_REFERENCE_KIND, TABLE_FUNCTION_REFERENCE_KIND}; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::main::quote_end::quote_end; @@ -14,8 +15,6 @@ const PREFIXES: [(&str, &str); 6] = [ ("__ref(", "ref"), ]; -pub(crate) type StaticReference = (String, String, Option, Option); - pub(crate) fn extract(sql: &str) -> Option> { let bytes = sql.as_bytes(); let mut references: Vec = Vec::new(); diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs new file mode 100644 index 0000000000..ae851a60d0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_references/mod.rs @@ -0,0 +1 @@ +pub mod extraction; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs new file mode 100644 index 0000000000..b06270fd75 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -0,0 +1,5 @@ +pub mod model_header_matching; +pub mod model_header_parsing; +pub mod model_header_tokenizing; +pub mod sql_interpolation; +pub mod sql_references; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_matching.rs similarity index 68% rename from crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs rename to crates/sqlbuild-sqltext/src/compiler/main/model_header_matching.rs index bbe71611bd..499865c4dc 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_matching.rs @@ -1,6 +1,6 @@ use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets}; -pub(crate) fn match_batch(contents: &[String]) -> Vec> { +pub fn match_batch(contents: &[String]) -> Vec> { contents .iter() .map(|text| matching::match_one(text)) diff --git a/crates/sqlbuild-rules-native/src/compiler/main/model_header_parsing.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_parsing.rs similarity index 58% rename from crates/sqlbuild-rules-native/src/compiler/main/model_header_parsing.rs rename to crates/sqlbuild-sqltext/src/compiler/main/model_header_parsing.rs index 98f5ee5a0e..8c67b62fc3 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/model_header_parsing.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_parsing.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::model_headers::tokenization::{self, HeaderParseResult}; -pub(crate) fn parse_batch(headers: &[String]) -> Result, String> { +pub fn parse_batch(headers: &[String]) -> Result, String> { tokenization::parse_batch(headers) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/model_header_tokenizing.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_tokenizing.rs similarity index 60% rename from crates/sqlbuild-rules-native/src/compiler/main/model_header_tokenizing.rs rename to crates/sqlbuild-sqltext/src/compiler/main/model_header_tokenizing.rs index 7536754eee..27724c39e6 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/model_header_tokenizing.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_tokenizing.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::model_headers::tokenization::{self, HeaderToken}; -pub(crate) fn tokenize_one(header: &str) -> Result, String> { +pub fn tokenize_one(header: &str) -> Result, String> { tokenization::tokenize_one(header) } diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs similarity index 89% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_interpolation.rs rename to crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs index 471d2be4b8..6ce7e114ff 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/sql_interpolation.rs @@ -1,6 +1,6 @@ //! Public compiler entry point for conservative scalar SQL interpolation. -pub(crate) fn substitute_batch( +pub fn substitute_batch( sqls: &[String], variables: &[(String, String)], ) -> Vec<(u8, Option)> { diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_references.rs b/crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs similarity index 51% rename from crates/sqlbuild-rules-native/src/compiler/main/sql_references.rs rename to crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs index e9110f3fed..d48a41008c 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/sql_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/sql_references.rs @@ -1,7 +1,7 @@ //! Public compiler entry point for conservative logical SQL references. -use crate::compiler::_helpers::sql_references::extraction::StaticReference; +use crate::compiler::types::StaticReference; -pub(crate) fn extract(sql: &str) -> Option> { +pub fn extract(sql: &str) -> Option> { crate::compiler::_helpers::sql_references::extraction::extract(sql) } diff --git a/crates/sqlbuild-sqltext/src/compiler/mod.rs b/crates/sqlbuild-sqltext/src/compiler/mod.rs new file mode 100644 index 0000000000..74f62c9e0a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/mod.rs @@ -0,0 +1,7 @@ +pub mod _helpers; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/compiler/models.rs b/crates/sqlbuild-sqltext/src/compiler/models.rs similarity index 69% rename from crates/sqlbuild-rules-native/src/compiler/models.rs rename to crates/sqlbuild-sqltext/src/compiler/models.rs index ef144dab88..3950629e32 100644 --- a/crates/sqlbuild-rules-native/src/compiler/models.rs +++ b/crates/sqlbuild-sqltext/src/compiler/models.rs @@ -1,5 +1,5 @@ #[derive(Clone, Debug, PartialEq)] -pub(crate) enum AuthoredValue { +pub enum AuthoredValue { Null, Boolean(bool), BareWord(String), @@ -13,9 +13,3 @@ pub(crate) enum AuthoredValue { NamedSqlHook(String, Vec<(String, AuthoredValue)>), PythonHook(String, Vec<(String, AuthoredValue)>), } - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct SqlTestFixtureFacts { - pub(crate) mock: bool, - pub(crate) empty_fixture_marker: bool, -} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests.rs b/crates/sqlbuild-sqltext/src/compiler/tests.rs new file mode 100644 index 0000000000..6b14cd92fe --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_model_header_matching.rs"] +mod model_header_matching; +#[path = "tests/test_model_header_tokenization.rs"] +mod model_header_tokenization; +#[path = "tests/test_sql_interpolation.rs"] +mod sql_interpolation; +#[path = "tests/test_sql_references.rs"] +mod sql_references; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs new file mode 100644 index 0000000000..84ca3c6112 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -0,0 +1,207 @@ +use crate::compiler::_helpers::model_headers::tokenization::{ + MAX_TOKENIZER_WORKERS, TOKENIZER_WORKER_STACK_BYTES, build_tokenizer_pool, parse_batch, +}; +use crate::compiler::_helpers::sql_interpolation::substitution::{ + FALLBACK, SUBSTITUTED, UNCHANGED, substitute_batch, +}; +use crate::compiler::_helpers::sql_references::extraction::extract; +use crate::compiler::models::AuthoredValue; + +pub(crate) fn scalar_variables_preserve_lexical_boundaries() -> bool { + let sqls = vec![ + "SELECT @@revision, '@@status', @@@window_start".to_owned(), + "-- @@revision\nSELECT /* @@status */ 1".to_owned(), + "SELECT '@@revision''s'".to_owned(), + ]; + substitute_batch( + &sqls, + &[ + ("revision".to_owned(), "7".to_owned()), + ("status".to_owned(), "ready".to_owned()), + ], + ) == vec![ + ( + SUBSTITUTED, + Some("SELECT 7, 'ready', @@@window_start".to_owned()), + ), + (UNCHANGED, None), + (SUBSTITUTED, Some("SELECT '7''s'".to_owned())), + ] +} + +pub(crate) fn dynamic_or_malformed_sql_requests_fallback() -> bool { + let sqls = vec![ + "SELECT @@ENV:USER".to_owned(), + "SELECT @@missing".to_owned(), + "SELECT @@revision, 'unterminated".to_owned(), + "SELECT @@revision /* unterminated".to_owned(), + "SELECT @@révision".to_owned(), + ]; + substitute_batch(&sqls, &[("revision".to_owned(), "7".to_owned())]) + == vec![(FALLBACK, None); sqls.len()] +} + +pub(crate) fn simple_references_preserve_authored_order() -> bool { + extract( + "SELECT * FROM __source('orders') UNION ALL SELECT * FROM __dbt_ref(\"shop\", \"customers\")", + ) == Some(vec![ + ("source".to_owned(), "orders".to_owned(), None, None), + ( + "dbt_ref".to_owned(), + "customers".to_owned(), + Some("shop".to_owned()), + None, + ), + ]) +} + +pub(crate) fn comments_and_quoted_text_hide_references() -> bool { + extract("-- __ref(\"ignored\")\nSELECT '__seed(\"also_ignored\")' FROM __ref(orders)") + == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) +} + +pub(crate) fn dollar_quoted_text_hides_references() -> bool { + extract(concat!( + "SELECT $$Customer's order -- __ref(\"ignored\") $5$$ AS label, ", + "$tag$ $$ __seed(\"also_ignored\") $tag$ AS note, price$1$ ", + "FROM __ref(orders)" + )) == Some(vec![("ref".to_owned(), "orders".to_owned(), None, None)]) +} + +pub(crate) fn complex_or_malformed_sql_requests_fallback() -> bool { + [ + "SELECT * FROM __table_fn(\"orders\")(1)", + "SELECT * FROM __ref(concat('ord', 'ers'))", + "SELECT * FROM __ref(\"orders\"", + "SELECT * FROM __ref(\"orders\") /* unterminated", + "SELECT * FROM __ref(\"orders\") WHERE note = 'unterminated", + "SELECT * FROM __ref(\"orders\") WHERE note = $$unterminated", + "SELECT * FROM __ref(örders)", + ] + .into_iter() + .all(|sql| extract(sql).is_none()) +} + +pub(crate) fn nested_authored_headers_preserve_values_and_offsets() -> bool { + let headers = vec![ + " columns (café (type DECIMAL(10,2))), enabled true".to_owned(), + "constants (_countries {FR, GB}, _size constant(value 2))".to_owned(), + ]; + let results = parse_batch(&headers).expect("worker pool builds"); + assert_eq!(results[0].2, None); + assert_eq!( + results[0].1.as_ref().expect("column offsets"), + &vec![("café".to_owned(), 10, 4)] + ); + assert_eq!( + results[0].0, + Some(AuthoredValue::Map(vec![ + ( + "columns".to_owned(), + AuthoredValue::Map(vec![( + "café".to_owned(), + AuthoredValue::Map(vec![( + "type".to_owned(), + AuthoredValue::String("DECIMAL(10,2)".to_owned()), + )]), + )]), + ), + ("enabled".to_owned(), AuthoredValue::Boolean(true)), + ])) + ); + assert!(matches!( + results[1].0, + Some(AuthoredValue::Map(ref values)) if values.len() == 1 + )); + true +} + +pub(crate) fn invalid_headers_return_each_exact_error_in_order() -> bool { + let headers = vec![ + "schema \"analytics".to_owned(), + "schema analytics\"mart".to_owned(), + "schema ${ENV".to_owned(), + ]; + let results = parse_batch(&headers).expect("worker pool builds"); + assert_eq!( + results + .iter() + .map(|result| result.2.as_deref()) + .collect::>(), + vec![ + Some("unterminated double-quoted string at position 7"), + Some("unexpected double quote inside bare value at position 16; quote the whole value"), + Some("unterminated template value at position 7"), + ] + ); + true +} + +pub(crate) fn header_comments_are_skipped_without_shifting_offsets() -> bool { + let headers = vec![ + "materialized table -- keep model metadata\n columns (id ()) /* it's, (fine) */" + .to_owned(), + "materialized table--trailing note\n".to_owned(), + "materialized table /* unclosed".to_owned(), + ]; + + let results = parse_batch(&headers).expect("worker pool builds"); + + let AuthoredValue::Map(values) = results[0].0.as_ref().expect("header values") else { + panic!("header values must be a map"); + }; + assert_eq!( + values + .iter() + .map(|(key, _)| key.as_str()) + .collect::>(), + vec!["materialized", "columns"] + ); + assert_eq!( + results[0].1.as_ref().expect("column offsets"), + &vec![("id".to_owned(), 53, 2)] + ); + let AuthoredValue::Map(values) = results[1].0.as_ref().expect("header values") else { + panic!("header values must be a map"); + }; + assert_eq!( + values + .iter() + .map(|(key, _)| key.as_str()) + .collect::>(), + vec!["materialized"] + ); + assert_eq!( + results[2].2.as_deref(), + Some("unterminated block comment at position 19") + ); + true +} + +pub(crate) fn nested_and_root_columns_return_only_root_offsets() -> bool { + let headers = vec![ + "config (columns (nested (type INTEGER))), columns (top (type INTEGER))".to_owned(), + "config (columns (nested (type INTEGER)))".to_owned(), + ]; + + let results = parse_batch(&headers).expect("worker pool builds"); + + assert_eq!( + results[0].1.as_ref().expect("column offsets"), + &vec![("top".to_owned(), 51, 3)] + ); + assert_eq!( + results[1].1.as_ref().expect("column offsets"), + &Vec::<(String, usize, usize)>::new() + ); + true +} + +pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { + for (header_count, expected_workers) in [(0, 1), (1, 1), (3, 3), (5, MAX_TOKENIZER_WORKERS)] { + let pool = build_tokenizer_pool(header_count).expect("worker pool builds"); + assert_eq!(pool.current_num_threads(), expected_workers); + } + assert_eq!(TOKENIZER_WORKER_STACK_BYTES, 16 * 1024 * 1024); + true +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_matching.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs rename to crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_matching.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_tokenization.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_tokenization.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_tokenization.rs rename to crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_tokenization.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_interpolation.rs rename to crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/compiler/tests/test_sql_references.rs rename to crates/sqlbuild-sqltext/src/compiler/tests/test_sql_references.rs diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs new file mode 100644 index 0000000000..269da1d49a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -0,0 +1,17 @@ +pub(crate) struct ModelHeaderTokenizationTestCase { + pub(crate) description: &'static str, + pub(crate) run: fn() -> bool, + pub(crate) expected_success: bool, +} + +pub(crate) struct StaticSqlOperationTestCase { + pub(crate) description: &'static str, + pub(crate) run: fn() -> bool, + pub(crate) expected_success: bool, +} + +pub(crate) struct ModelHeaderMatchTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) expected_offsets: Option<(usize, usize, usize)>, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs new file mode 100644 index 0000000000..f5f935751a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -0,0 +1 @@ +pub type StaticReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-sqltext/src/constants.rs b/crates/sqlbuild-sqltext/src/constants.rs new file mode 100644 index 0000000000..0025aeadcf --- /dev/null +++ b/crates/sqlbuild-sqltext/src/constants.rs @@ -0,0 +1,9 @@ +pub(crate) const COLUMNS_KEY: &str = "columns"; +pub(crate) const CONSTANT_CALL: &str = "constant"; +pub(crate) const DBT_REFERENCE_KIND: &str = "dbt_ref"; +pub(crate) const INLINE_SQL_HOOK: &str = "inline_sql"; +pub(crate) const OUTSIDE_KEY: &str = "outside"; +pub(crate) const SQL_HOOK: &str = "sql"; +pub(crate) const TABLE_FUNCTION_REFERENCE_KIND: &str = "table_function"; +pub(crate) const THRESHOLDS_KEY: &str = "thresholds"; +pub(crate) const TYPE_KEY: &str = "type"; diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs new file mode 100644 index 0000000000..1f02828348 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -0,0 +1,7 @@ +//! Lexical SQL text: scanning, model headers, static variable substitution and references. + +#![forbid(unsafe_code)] + +pub mod compiler; +mod constants; +pub mod sql_scan; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/dialect.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs rename to crates/sqlbuild-sqltext/src/sql_scan/_helpers/dialect.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/dollar.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs rename to crates/sqlbuild-sqltext/src/sql_scan/_helpers/dollar.rs diff --git a/crates/sqlbuild-sqltext/src/sql_scan/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/mod.rs new file mode 100644 index 0000000000..2f870bffee --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub mod dialect; +pub mod dollar; +pub mod parens; +pub(crate) mod rules_dialect; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/parens.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs rename to crates/sqlbuild-sqltext/src/sql_scan/_helpers/parens.rs diff --git a/crates/sqlbuild-sqltext/src/sql_scan/_helpers/rules_dialect.rs b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/rules_dialect.rs new file mode 100644 index 0000000000..e98fe2284d --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/_helpers/rules_dialect.rs @@ -0,0 +1,32 @@ +//! Lexical quoting facts of the SQL dialects the rules engine scans. + +use crate::sql_scan::models::QuotePolicy; +use sqlparser::dialect::{ + BigQueryDialect, ClickHouseDialect, DatabricksDialect, Dialect, DuckDbDialect, GenericDialect, + MsSqlDialect, PostgreSqlDialect, SnowflakeDialect, +}; + +pub(crate) fn rules_quote_policy(dialect_name: &str) -> QuotePolicy { + let dialect = rules_dialect(dialect_name); + let backslash_escapes = dialect.supports_string_literal_backslash_escape(); + QuotePolicy { + backtick_identifiers: dialect.is_delimited_identifier_start('`'), + single_quote_backslash_escapes: backslash_escapes, + double_quote_backslash_escapes: backslash_escapes + && !dialect.is_delimited_identifier_start('"'), + dollar_quotes: false, + } +} + +pub(crate) fn rules_dialect(name: &str) -> Box { + match name.to_ascii_lowercase().as_str() { + "bigquery" => Box::new(BigQueryDialect {}), + "clickhouse" => Box::new(ClickHouseDialect {}), + "databricks" => Box::new(DatabricksDialect {}), + "duckdb" => Box::new(DuckDbDialect {}), + "postgres" | "postgresql" => Box::new(PostgreSqlDialect {}), + "mssql" | "sqlserver" | "tsql" => Box::new(MsSqlDialect {}), + "snowflake" => Box::new(SnowflakeDialect {}), + _ => Box::new(GenericDialect {}), + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/constants.rs b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/constants.rs rename to crates/sqlbuild-sqltext/src/sql_scan/constants.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/comment_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/comment_end.rs similarity index 88% rename from crates/sqlbuild-rules-native/src/sql_scan/main/comment_end.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/comment_end.rs index b90d31b65d..e951e1689f 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/comment_end.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/comment_end.rs @@ -3,7 +3,7 @@ use crate::sql_scan::models::Unclosed; /// Return the end of a line or block comment starting at `index`, if one starts there. -pub(crate) fn comment_end(sql: &[u8], index: usize) -> Result, Unclosed> { +pub fn comment_end(sql: &[u8], index: usize) -> Result, Unclosed> { match (sql.get(index), sql.get(index + 1)) { (Some(b'-'), Some(b'-')) => Ok(Some( sql[index + 2..] diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_matching_paren.rs similarity index 90% rename from crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/dialect_matching_paren.rs index 687a007d86..9be140677d 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_matching_paren.rs @@ -4,7 +4,7 @@ use crate::sql_scan::_helpers::dialect::matching_paren; use crate::sql_scan::models::{LexicalSyntax, Unclosed}; /// Return the index of the parenthesis closing the one at `open` under the dialect's rules. -pub(crate) fn dialect_matching_paren( +pub fn dialect_matching_paren( sql: &[u8], open: usize, syntax: &LexicalSyntax, diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_ranges.rs similarity index 75% rename from crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_ranges.rs index 321597f648..cf7bc585be 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_non_code_ranges.rs @@ -4,6 +4,6 @@ use crate::sql_scan::_helpers::dialect::non_code_ranges; use crate::sql_scan::models::LexicalSyntax; /// Return every comment and quoted-text range in `sql`; an unclosed one extends to the end. -pub(crate) fn dialect_non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { +pub fn dialect_non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { non_code_ranges(sql, syntax) } diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/matching_paren.rs similarity index 77% rename from crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/matching_paren.rs index 363e245e82..d9bf713699 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/matching_paren.rs @@ -5,10 +5,6 @@ use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; /// Return the index of the parenthesis closing the one at `open`, skipping quotes and comments. -pub(crate) fn matching_paren( - sql: &[u8], - open: usize, - policy: QuotePolicy, -) -> Result { +pub fn matching_paren(sql: &[u8], open: usize, policy: QuotePolicy) -> Result { matching_paren_with(sql, open, |sql, index| non_code_end(sql, index, policy)) } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs new file mode 100644 index 0000000000..dcd98888be --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -0,0 +1,9 @@ +pub mod comment_end; +pub mod dialect_matching_paren; +pub mod dialect_non_code_ranges; +pub mod matching_paren; +pub mod non_code_end; +pub mod quote_end; +pub mod quote_policy; +pub mod rules_dialect; +pub mod skip_whitespace; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs similarity index 96% rename from crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs index 8fbd9ba3d7..8e8bdaf663 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs @@ -6,7 +6,7 @@ use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; /// Return the end of a comment or quoted text starting at `index`, if one starts there. -pub(crate) fn non_code_end( +pub fn non_code_end( sql: &[u8], index: usize, policy: QuotePolicy, diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/quote_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/quote_end.rs similarity index 87% rename from crates/sqlbuild-rules-native/src/sql_scan/main/quote_end.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/quote_end.rs index fdf256cea6..4d58601421 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/quote_end.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/quote_end.rs @@ -3,7 +3,7 @@ use crate::sql_scan::models::{QuotePolicy, Unclosed}; /// Return the index after the quoted text whose opening quote is at `start`. -pub(crate) fn quote_end(sql: &[u8], start: usize, policy: QuotePolicy) -> Result { +pub fn quote_end(sql: &[u8], start: usize, policy: QuotePolicy) -> Result { let quote = sql[start]; let backslash_escapes = policy.backslash_escapes(quote); let mut index = start + 1; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/quote_policy.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/quote_policy.rs new file mode 100644 index 0000000000..53bb85b521 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/quote_policy.rs @@ -0,0 +1,4 @@ +/// Return the quoting policy rules use when scanning SQL for `dialect_name`. +pub fn quote_policy(dialect_name: &str) -> crate::sql_scan::models::QuotePolicy { + crate::sql_scan::_helpers::rules_dialect::rules_quote_policy(dialect_name) +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/rules_dialect.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/rules_dialect.rs new file mode 100644 index 0000000000..402b035dda --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/rules_dialect.rs @@ -0,0 +1,6 @@ +use sqlparser::dialect::Dialect; + +/// Return the parser dialect rules use for `name`, falling back to the generic dialect. +pub fn rules_dialect(name: &str) -> Box { + crate::sql_scan::_helpers::rules_dialect::rules_dialect(name) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/skip_whitespace.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/skip_whitespace.rs similarity index 83% rename from crates/sqlbuild-rules-native/src/sql_scan/main/skip_whitespace.rs rename to crates/sqlbuild-sqltext/src/sql_scan/main/skip_whitespace.rs index a60ea5e9c4..b63e660c1a 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/skip_whitespace.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/skip_whitespace.rs @@ -1,7 +1,7 @@ //! Unicode whitespace skipping for SQL scanners. /// Return the first index at or after `index` that is not Unicode whitespace. -pub(crate) fn skip_whitespace(sql: &str, mut index: usize) -> usize { +pub fn skip_whitespace(sql: &str, mut index: usize) -> usize { while let Some(character) = sql.get(index..).and_then(|rest| rest.chars().next()) { if !character.is_whitespace() { break; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/mod.rs new file mode 100644 index 0000000000..8dbbf8187a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/mod.rs @@ -0,0 +1,7 @@ +pub mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-sqltext/src/sql_scan/models.rs similarity index 77% rename from crates/sqlbuild-rules-native/src/sql_scan/models.rs rename to crates/sqlbuild-sqltext/src/sql_scan/models.rs index fa699b1e68..632b1b5b07 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/models.rs @@ -2,7 +2,7 @@ /// The SQL construct a scan reached the end of input inside. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Unclosed { +pub enum Unclosed { BlockComment, Quote, Parenthesis, @@ -10,20 +10,20 @@ pub(crate) enum Unclosed { /// Dialect-dependent quoting rules; comments and doubled-quote escapes are fixed for every policy. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct QuotePolicy { +pub struct QuotePolicy { /// Backtick-delimited identifiers are quoted text. - pub(crate) backtick_identifiers: bool, + pub backtick_identifiers: bool, /// A backslash escapes the next byte inside single-quoted strings. - pub(crate) single_quote_backslash_escapes: bool, + pub single_quote_backslash_escapes: bool, /// A backslash escapes the next byte inside double-quoted text. - pub(crate) double_quote_backslash_escapes: bool, + pub double_quote_backslash_escapes: bool, /// `$$` and `$tag$` delimit opaque string literals. - pub(crate) dollar_quotes: bool, + pub dollar_quotes: bool, } impl QuotePolicy { /// Compiler policy, mirroring Python `sql_analysis`: backticks and dollar quotes are text. - pub(crate) const COMPILER: Self = Self { + pub const COMPILER: Self = Self { backtick_identifiers: true, single_quote_backslash_escapes: false, double_quote_backslash_escapes: false, @@ -31,7 +31,7 @@ impl QuotePolicy { }; /// SQL lint policy: backslashes escape, backticks per dialect, dollar quotes as in the compiler. - pub(crate) const SQL_LINT: Self = Self { + pub const SQL_LINT: Self = Self { backtick_identifiers: false, single_quote_backslash_escapes: true, double_quote_backslash_escapes: true, @@ -39,7 +39,7 @@ impl QuotePolicy { }; /// Return this policy with backtick-delimited identifiers enabled or disabled. - pub(crate) const fn with_backtick_identifiers(self, backtick_identifiers: bool) -> Self { + pub const fn with_backtick_identifiers(self, backtick_identifiers: bool) -> Self { Self { backtick_identifiers, ..self @@ -47,7 +47,7 @@ impl QuotePolicy { } /// Return whether `byte` opens quoted text under this policy. - pub(crate) fn is_quote(self, byte: u8) -> bool { + pub fn is_quote(self, byte: u8) -> bool { match byte { b'\'' | b'"' => true, b'`' => self.backtick_identifiers, @@ -68,19 +68,19 @@ impl QuotePolicy { /// One adapter's rules for where quoted text and comments end, mirroring Python `SqlLexicalSyntax`. #[derive(Clone, Debug, Default, PartialEq, Eq, serde::Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub(crate) struct LexicalSyntax { +pub struct LexicalSyntax { /// Quote characters inside which a backslash escapes the next byte. - pub(crate) backslash_escape_quotes: Vec, + pub backslash_escape_quotes: Vec, /// `E'...'` strings honour backslash escapes. - pub(crate) escape_string_prefix: bool, + pub escape_string_prefix: bool, /// `r'...'` strings disable backslash escapes. - pub(crate) raw_string_prefix: bool, + pub raw_string_prefix: bool, /// `'''...'''` and `"""..."""` delimit strings. - pub(crate) triple_quoted_strings: bool, + pub triple_quoted_strings: bool, /// Block comments nest. - pub(crate) nested_block_comments: bool, + pub nested_block_comments: bool, /// Prefixes that start a line comment, such as `--`, `#` or `//`. - pub(crate) line_comment_prefixes: Vec, + pub line_comment_prefixes: Vec, } impl LexicalSyntax { diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/tests.rs rename to crates/sqlbuild-sqltext/src/sql_scan/tests.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs rename to crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs rename to crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/tests/test_sql_scan.rs rename to crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs similarity index 100% rename from crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs rename to crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs diff --git a/fensu.toml b/fensu.toml index e855b92b7e..fc7cc68f9e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -257,24 +257,34 @@ select = ["FPRS"] [[targets.rust.rule_exceptions]] rule = "FPRSR307" -path = "crates/sqlbuild-rules-native/src/scope_metadata/main.rs" +path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" reason = "This focused serde boundary validates the scope metadata schema used by the crate-root wire models." [[targets.rust.rule_exceptions]] rule = "FPRSR307" -path = "crates/sqlbuild-rules-native/src/semantic_usage/main.rs" +path = "crates/sqlbuild-analysis/src/semantic_usage/main.rs" reason = "This module owns the existing coarse native SQL usage-analysis boundary exposed through PyO3." [[targets.rust.rule_exceptions]] rule = "FPRSR307" -path = "crates/sqlbuild-rules-native/src/semantic_validation/main.rs" +path = "crates/sqlbuild-analysis/src/semantic_validation/main.rs" reason = "This module owns the existing coarse native schema-validation boundary exposed through PyO3." [[targets.rust.rule_exceptions]] rule = "FPRSS201" -path = "crates/sqlbuild-rules-native/src/semantic_validation/models.rs" +path = "crates/sqlbuild-analysis/src/semantic_validation/models.rs" reason = "ProjectCatalog implements Debug manually because Polyglot SchemaValidationOptions does not implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python/src/bindings/models.rs" +reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs" +reason = "This test-support entry exists only for the SQL lint tests in sqlbuild-rules, which the analyzer does not count as importers." + [targets.rust.rule_options.FPRSL301] forbidden_packages = [ "sqlbuild-structure-checker", @@ -283,12 +293,16 @@ forbidden_packages = [ [targets.rust.rule_options.FPRSL102] packages = [] -restricted_paths = ["crates/sqlbuild-rules-native/src/rules"] +restricted_paths = ["crates/sqlbuild-rules/src/rules"] remediation = "consume SQLBuild-owned SQL fact rows instead of raw parser types" [targets.rust.rule_options.FPRSL304] crate_names = [ - "sqlbuild-rules-native", + "sqlbuild-analysis", + "sqlbuild-core", + "sqlbuild-python", + "sqlbuild-rules", + "sqlbuild-sqltext", ] [[dupes.contract_exemptions]] diff --git a/pyproject.toml b/pyproject.toml index 828dcd752a..148976d6ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ dev = [ ] [tool.maturin] -manifest-path = "crates/sqlbuild-rules-native/Cargo.toml" +manifest-path = "crates/sqlbuild-python/Cargo.toml" module-name = "sqlbuild._native" python-source = "src" features = ["extension-module"] @@ -138,8 +138,8 @@ package = true cache-keys = [ { file = "pyproject.toml" }, { file = "Cargo.toml" }, - { file = "crates/sqlbuild-rules-native/Cargo.toml" }, - { file = "crates/sqlbuild-rules-native/build.rs" }, - { file = "crates/sqlbuild-rules-native/src/**/*.rs" }, + { file = "crates/*/Cargo.toml" }, + { file = "crates/*/build.rs" }, + { file = "crates/*/src/**/*.rs" }, { file = "src/sqlbuild/_native.pyi" }, ] diff --git a/release-please-config.json b/release-please-config.json index 619d73104e..daed61d50d 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -67,8 +67,8 @@ "extra-files": [ { "type": "toml", - "path": "crates/sqlbuild-rules-native/Cargo.toml", - "jsonpath": "$.package.version" + "path": "Cargo.toml", + "jsonpath": "$.workspace.package.version" }, { "type": "toml", diff --git a/scripts/check_native_layering.py b/scripts/check_native_layering.py new file mode 100644 index 0000000000..69fa1071db --- /dev/null +++ b/scripts/check_native_layering.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the native crate layering check.""" + +from __future__ import annotations + +from scripts.native_layering.main.check import run_native_layering_check + + +def main(argv: list[str] | None = None) -> int: + """Check native crate layering.""" + return run_native_layering_check(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/native_layering/__init__.py b/scripts/native_layering/__init__.py new file mode 100644 index 0000000000..f0ed452c84 --- /dev/null +++ b/scripts/native_layering/__init__.py @@ -0,0 +1 @@ +"""Native Rust workspace layering checks.""" diff --git a/scripts/native_layering/_helpers/__init__.py b/scripts/native_layering/_helpers/__init__.py new file mode 100644 index 0000000000..24983096c6 --- /dev/null +++ b/scripts/native_layering/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native layering helpers.""" diff --git a/scripts/native_layering/_helpers/layering.py b/scripts/native_layering/_helpers/layering.py new file mode 100644 index 0000000000..4de1f8c577 --- /dev/null +++ b/scripts/native_layering/_helpers/layering.py @@ -0,0 +1,133 @@ +"""Read the native Cargo workspace and report crate layering violations.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +from scripts.native_layering.constants import ( + DEPENDENCY_TABLES, + LAYERS_METADATA_KEY, + POLYGLOT_PACKAGE, + PYO3_PACKAGE, +) +from scripts.native_layering.models import NativeLayers + + +def get_native_layering_errors(root: Path) -> tuple[str, ...]: + """Return every layering violation of the Cargo workspace rooted at `root`.""" + workspace: dict[str, Any] = _read_toml(root / "Cargo.toml") + layers: NativeLayers | None = _declared_layers(workspace) + if layers is None: + return (f"Cargo.toml must declare [workspace.metadata.{LAYERS_METADATA_KEY}].",) + members: dict[str, set[str]] = _member_dependencies(root=root, workspace=workspace) + errors: list[str] = _order_errors(layers=layers, members=members) + if errors: + return tuple(errors) + rank: dict[str, int] = {name: index for index, name in enumerate(layers.order)} + for name in layers.order: + for dependency in sorted(members[name] & members.keys()): + if rank[dependency] >= rank[name]: + errors.append(f"{name} depends on {dependency}, which is not in an earlier layer.") + lock_graph: dict[str, set[str]] = _lock_graph(root / "Cargo.lock") + for name in layers.order: + reachable: set[str] = _reachable( + direct=members[name] | lock_graph.get(name, set()), graph=lock_graph + ) + if name != layers.python_boundary and _reaches(reachable=reachable, package=PYO3_PACKAGE): + errors.append(f"{name} depends on {PYO3_PACKAGE}; only {layers.python_boundary} may.") + if rank[name] < rank[layers.polyglot_floor] and _reaches( + reachable=reachable, package=POLYGLOT_PACKAGE + ): + errors.append( + f"{name} depends on {POLYGLOT_PACKAGE}, which starts at {layers.polyglot_floor}." + ) + return tuple(errors) + + +def _declared_layers(workspace: dict[str, Any]) -> NativeLayers | None: + metadata: Any = workspace.get("workspace", {}).get("metadata", {}).get(LAYERS_METADATA_KEY) + if not isinstance(metadata, dict): + return None + return NativeLayers( + order=tuple(metadata.get("order", ())), + python_boundary=str(metadata.get("python-boundary", "")), + polyglot_floor=str(metadata.get("polyglot-floor", "")), + ) + + +def _order_errors(*, layers: NativeLayers, members: dict[str, set[str]]) -> list[str]: + errors: list[str] = [] + if len(set(layers.order)) != len(layers.order): + errors.append("The native layer order lists a crate more than once.") + for name in sorted(members.keys() - set(layers.order)): + errors.append(f"Workspace crate {name} is missing from the native layer order.") + for name in sorted(set(layers.order) - members.keys()): + errors.append(f"Native layer order lists {name}, which is not a workspace crate.") + for role, name in ( + ("python-boundary", layers.python_boundary), + ("polyglot-floor", layers.polyglot_floor), + ): + if name not in layers.order: + errors.append(f"Native layer {role} {name!r} is not in the layer order.") + return errors + + +def _member_dependencies(*, root: Path, workspace: dict[str, Any]) -> dict[str, set[str]]: + members: dict[str, set[str]] = {} + shared: dict[str, Any] = workspace.get("workspace", {}).get("dependencies", {}) + for pattern in workspace.get("workspace", {}).get("members", ()): + for directory in sorted(root.glob(pattern)): + manifest: dict[str, Any] = _read_toml(directory / "Cargo.toml") + members[manifest["package"]["name"]] = _direct_dependencies( + manifest=manifest, shared=shared + ) + return members + + +def _direct_dependencies(*, manifest: dict[str, Any], shared: dict[str, Any]) -> set[str]: + tables: list[dict[str, Any]] = [manifest.get(table, {}) for table in DEPENDENCY_TABLES] + for target in manifest.get("target", {}).values(): + tables.extend(target.get(table, {}) for table in DEPENDENCY_TABLES) + names: set[str] = set() + for table in tables: + for key, spec in table.items(): + names.add(_package_name(key=key, spec=spec, shared=shared)) + return names + + +def _package_name(*, key: str, spec: Any, shared: dict[str, Any]) -> str: + """Resolve a dependency key to its package, following `package` renames and inheritance.""" + if isinstance(spec, dict) and spec.get("workspace") is True: + spec = shared.get(key) + renamed: Any = spec.get("package") if isinstance(spec, dict) else None + return renamed if isinstance(renamed, str) else key + + +def _lock_graph(lockfile: Path) -> dict[str, set[str]]: + graph: dict[str, set[str]] = {} + for package in _read_toml(lockfile).get("package", ()): + dependencies: set[str] = {entry.split(" ")[0] for entry in package.get("dependencies", ())} + graph.setdefault(package["name"], set()).update(dependencies) + return graph + + +def _reachable(*, direct: set[str], graph: dict[str, set[str]]) -> set[str]: + seen: set[str] = set() + pending: list[str] = list(direct) + while pending: + current: str = pending.pop() + if current not in seen: + seen.add(current) + pending.extend(graph.get(current, ())) + return seen + + +def _reaches(*, reachable: set[str], package: str) -> bool: + return any(name == package or name.startswith(f"{package}-") for name in reachable) + + +def _read_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + return tomllib.load(handle) diff --git a/scripts/native_layering/_helpers/versions.py b/scripts/native_layering/_helpers/versions.py new file mode 100644 index 0000000000..c9c05d26c2 --- /dev/null +++ b/scripts/native_layering/_helpers/versions.py @@ -0,0 +1,35 @@ +"""Check that every release version source names the same version.""" + +from __future__ import annotations + +import json +import tomllib +from pathlib import Path +from typing import Any + +from scripts.native_layering.constants import ( + CARGO_MANIFEST, + PYPROJECT_MANIFEST, + RELEASE_MANIFEST, + RELEASE_MANIFEST_PACKAGE, +) + + +def get_native_version_errors(root: Path) -> tuple[str, ...]: + """Return an error when the Cargo workspace, Python package and release versions differ.""" + with (root / CARGO_MANIFEST).open("rb") as handle: + cargo: dict[str, Any] = tomllib.load(handle) + with (root / PYPROJECT_MANIFEST).open("rb") as handle: + pyproject: dict[str, Any] = tomllib.load(handle) + release: dict[str, Any] = json.loads((root / RELEASE_MANIFEST).read_text(encoding="utf-8")) + versions: dict[str, Any] = { + f"{CARGO_MANIFEST} [workspace.package] version": cargo.get("workspace", {}) + .get("package", {}) + .get("version"), + f"{PYPROJECT_MANIFEST} [project] version": pyproject.get("project", {}).get("version"), + f"{RELEASE_MANIFEST} version": release.get(RELEASE_MANIFEST_PACKAGE), + } + if len(set(versions.values())) == 1: + return () + listed: str = ", ".join(f"{source} is {version}" for source, version in versions.items()) + return (f"Release versions differ: {listed}.",) diff --git a/scripts/native_layering/constants.py b/scripts/native_layering/constants.py new file mode 100644 index 0000000000..6cf6a3b6f8 --- /dev/null +++ b/scripts/native_layering/constants.py @@ -0,0 +1,10 @@ +"""Stable native layering policy constants.""" + +LAYERS_METADATA_KEY: str = "native-layers" +PYO3_PACKAGE: str = "pyo3" +POLYGLOT_PACKAGE: str = "polyglot-sql" +DEPENDENCY_TABLES: tuple[str, ...] = ("dependencies", "build-dependencies", "dev-dependencies") +CARGO_MANIFEST: str = "Cargo.toml" +PYPROJECT_MANIFEST: str = "pyproject.toml" +RELEASE_MANIFEST: str = ".release-please-manifest.json" +RELEASE_MANIFEST_PACKAGE: str = "." diff --git a/scripts/native_layering/main/__init__.py b/scripts/native_layering/main/__init__.py new file mode 100644 index 0000000000..70003e79b3 --- /dev/null +++ b/scripts/native_layering/main/__init__.py @@ -0,0 +1 @@ +"""Native layering command entrypoints.""" diff --git a/scripts/native_layering/main/check.py b/scripts/native_layering/main/check.py new file mode 100644 index 0000000000..4c4e6a2711 --- /dev/null +++ b/scripts/native_layering/main/check.py @@ -0,0 +1,27 @@ +"""Native crate layering command entrypoint.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.native_layering._helpers.layering import get_native_layering_errors +from scripts.native_layering._helpers.versions import get_native_version_errors + + +def run_native_layering_check(argv: list[str] | None = None) -> int: + """Fail when a native crate breaks the declared layering or the release versions differ.""" + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + arguments: argparse.Namespace = parser.parse_args(argv) + errors: tuple[str, ...] = get_native_layering_errors( + arguments.root + ) + get_native_version_errors(arguments.root) + if errors: + print("Native crate layering is invalid:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Native crate layering is valid.") + return 0 diff --git a/scripts/native_layering/models.py b/scripts/native_layering/models.py new file mode 100644 index 0000000000..c9e191ba22 --- /dev/null +++ b/scripts/native_layering/models.py @@ -0,0 +1,14 @@ +"""Native layering policy models.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeLayers: + """Declared crate order and the crates that bound PyO3 and polyglot.""" + + order: tuple[str, ...] + python_boundary: str + polyglot_floor: str diff --git a/tests/unit/scripts/native_layering/__init__.py b/tests/unit/scripts/native_layering/__init__.py new file mode 100644 index 0000000000..714d2ef8eb --- /dev/null +++ b/tests/unit/scripts/native_layering/__init__.py @@ -0,0 +1 @@ +"""Native layering tooling tests.""" diff --git a/tests/unit/scripts/native_layering/_helpers/__init__.py b/tests/unit/scripts/native_layering/_helpers/__init__.py new file mode 100644 index 0000000000..74dcdd7af6 --- /dev/null +++ b/tests/unit/scripts/native_layering/_helpers/__init__.py @@ -0,0 +1 @@ +"""Native layering helper tests.""" diff --git a/tests/unit/scripts/native_layering/_helpers/_test_types.py b/tests/unit/scripts/native_layering/_helpers/_test_types.py new file mode 100644 index 0000000000..e4884e89b7 --- /dev/null +++ b/tests/unit/scripts/native_layering/_helpers/_test_types.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class NativeLayeringTestCase: + description: str + crate_dependencies: dict[str, tuple[str, ...]] + external_dependencies: dict[str, tuple[str, ...]] + order: tuple[str, ...] + expected_errors: tuple[str, ...] + workspace_aliases: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RepositoryLayeringTestCase: + description: str + expected_errors: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeVersionTestCase: + description: str + workspace_version: str + pyproject_version: str + release_version: str + expected_errors: tuple[str, ...] diff --git a/tests/unit/scripts/native_layering/_helpers/helpers.py b/tests/unit/scripts/native_layering/_helpers/helpers.py new file mode 100644 index 0000000000..5b37019fa1 --- /dev/null +++ b/tests/unit/scripts/native_layering/_helpers/helpers.py @@ -0,0 +1,55 @@ +from pathlib import Path + +from tests.unit.scripts.native_layering._helpers._test_types import ( + NativeLayeringTestCase, + NativeVersionTestCase, +) + + +def write_workspace(root: Path, test_case: NativeLayeringTestCase) -> None: + """Write a minimal Cargo workspace and lockfile shaped like the native crates.""" + members: str = ", ".join(f'"crates/{name}"' for name in test_case.crate_dependencies) + order: str = ", ".join(f'"{name}"' for name in test_case.order) + aliases: str = "".join( + f'{alias} = {{ path = "crates/{package}", package = "{package}" }}\n' + for alias, package in test_case.workspace_aliases.items() + ) + (root / "Cargo.toml").write_text( + f"[workspace]\nmembers = [{members}]\n\n" + "[workspace.metadata.native-layers]\n" + f"order = [{order}]\n" + 'python-boundary = "sqlbuild-python"\n' + 'polyglot-floor = "sqlbuild-analysis"\n\n' + f"[workspace.dependencies]\n{aliases}", + encoding="utf-8", + ) + lock: list[str] = ["version = 4\n"] + for name, dependencies in test_case.crate_dependencies.items(): + crate: Path = root / "crates" / name + crate.mkdir(parents=True) + listed: str = "".join(f"{dependency}.workspace = true\n" for dependency in dependencies) + (crate / "Cargo.toml").write_text( + f'[package]\nname = "{name}"\n\n[dependencies]\n{listed}', encoding="utf-8" + ) + lock.append(_lock_package(name, dependencies)) + for name, dependencies in test_case.external_dependencies.items(): + lock.append(_lock_package(name, dependencies)) + (root / "Cargo.lock").write_text("\n".join(lock), encoding="utf-8") + + +def _lock_package(name: str, dependencies: tuple[str, ...]) -> str: + listed: str = ", ".join(f'"{dependency}"' for dependency in dependencies) + return f'[[package]]\nname = "{name}"\nversion = "0.1.0"\ndependencies = [{listed}]\n' + + +def write_versions(root: Path, test_case: NativeVersionTestCase) -> None: + """Write the three release version sources the version check compares.""" + (root / "Cargo.toml").write_text( + f'[workspace.package]\nversion = "{test_case.workspace_version}"\n', encoding="utf-8" + ) + (root / "pyproject.toml").write_text( + f'[project]\nversion = "{test_case.pyproject_version}"\n', encoding="utf-8" + ) + (root / ".release-please-manifest.json").write_text( + f'{{".": "{test_case.release_version}"}}\n', encoding="utf-8" + ) diff --git a/tests/unit/scripts/native_layering/_helpers/test_layering.py b/tests/unit/scripts/native_layering/_helpers/test_layering.py new file mode 100644 index 0000000000..4a26066d9b --- /dev/null +++ b/tests/unit/scripts/native_layering/_helpers/test_layering.py @@ -0,0 +1,133 @@ +from pathlib import Path + +import pytest + +from scripts.native_layering._helpers.layering import get_native_layering_errors +from tests.unit.scripts.native_layering._helpers._test_types import ( + NativeLayeringTestCase, + RepositoryLayeringTestCase, +) +from tests.unit.scripts.native_layering._helpers.helpers import write_workspace + +ORDER: tuple[str, ...] = ( + "sqlbuild-core", + "sqlbuild-sqltext", + "sqlbuild-analysis", + "sqlbuild-python", +) +LAYERED: dict[str, tuple[str, ...]] = { + "sqlbuild-core": (), + "sqlbuild-sqltext": ("sqlbuild-core",), + "sqlbuild-analysis": ("sqlbuild-core", "sqlbuild-sqltext", "polyglot-sql"), + "sqlbuild-python": ("sqlbuild-analysis", "pyo3"), +} +EXTERNAL: dict[str, tuple[str, ...]] = { + "polyglot-sql": ("serde",), + "pyo3": ("pyo3-ffi",), + "pyo3-ffi": (), + "serde": (), +} + + +@pytest.mark.parametrize( + "test_case", + [ + NativeLayeringTestCase( + description="one-way layers with PyO3 only in the Python crate", + crate_dependencies=LAYERED, + external_dependencies=EXTERNAL, + order=ORDER, + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_layered_workspace_when_checking_then_returns_no_errors( + tmp_path: Path, test_case: NativeLayeringTestCase +) -> None: + write_workspace(tmp_path, test_case) + + assert get_native_layering_errors(tmp_path) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + NativeLayeringTestCase( + description="lower crate depends on a later layer", + crate_dependencies={**LAYERED, "sqlbuild-core": ("sqlbuild-sqltext",)}, + external_dependencies=EXTERNAL, + order=ORDER, + expected_errors=( + "sqlbuild-core depends on sqlbuild-sqltext, which is not in an earlier layer.", + ), + ), + NativeLayeringTestCase( + description="analysis reaches PyO3 through an external crate", + crate_dependencies={ + **LAYERED, + "sqlbuild-analysis": ("sqlbuild-core", "polyglot-sql", "python-bridge"), + }, + external_dependencies={**EXTERNAL, "python-bridge": ("pyo3",)}, + order=ORDER, + expected_errors=("sqlbuild-analysis depends on pyo3; only sqlbuild-python may.",), + ), + NativeLayeringTestCase( + description="crate below the polyglot floor uses polyglot", + crate_dependencies={**LAYERED, "sqlbuild-sqltext": ("sqlbuild-core", "polyglot-sql")}, + external_dependencies=EXTERNAL, + order=ORDER, + expected_errors=( + "sqlbuild-sqltext depends on polyglot-sql, which starts at sqlbuild-analysis.", + ), + ), + NativeLayeringTestCase( + description="aliased workspace dependency points upward", + crate_dependencies={**LAYERED, "sqlbuild-core": ("text",)}, + external_dependencies=EXTERNAL, + order=ORDER, + workspace_aliases={"text": "sqlbuild-sqltext"}, + expected_errors=( + "sqlbuild-core depends on sqlbuild-sqltext, which is not in an earlier layer.", + ), + ), + NativeLayeringTestCase( + description="workspace crate missing from the layer order", + crate_dependencies={**LAYERED, "sqlbuild-extra": ("sqlbuild-core",)}, + external_dependencies=EXTERNAL, + order=ORDER, + expected_errors=( + "Workspace crate sqlbuild-extra is missing from the native layer order.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_layering_violation_when_checking_then_reports_it( + tmp_path: Path, test_case: NativeLayeringTestCase +) -> None: + write_workspace(tmp_path, test_case) + + assert get_native_layering_errors(tmp_path) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryLayeringTestCase( + description="repository crates follow their declared layers", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_workspace_when_checking_then_layers_hold( + test_case: RepositoryLayeringTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[5] + + assert get_native_layering_errors(repository) == test_case.expected_errors + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/native_layering/_helpers/test_versions.py b/tests/unit/scripts/native_layering/_helpers/test_versions.py new file mode 100644 index 0000000000..24652c0d13 --- /dev/null +++ b/tests/unit/scripts/native_layering/_helpers/test_versions.py @@ -0,0 +1,78 @@ +from pathlib import Path + +import pytest + +from scripts.native_layering._helpers.versions import get_native_version_errors +from tests.unit.scripts.native_layering._helpers._test_types import ( + NativeVersionTestCase, + RepositoryLayeringTestCase, +) +from tests.unit.scripts.native_layering._helpers.helpers import write_versions + + +@pytest.mark.parametrize( + "test_case", + [ + NativeVersionTestCase( + description="every release version source agrees", + workspace_version="1.2.0", + pyproject_version="1.2.0", + release_version="1.2.0", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_matching_versions_when_checking_then_returns_no_errors( + tmp_path: Path, test_case: NativeVersionTestCase +) -> None: + write_versions(tmp_path, test_case) + + assert get_native_version_errors(tmp_path) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + NativeVersionTestCase( + description="workspace version lags a release", + workspace_version="1.1.9", + pyproject_version="1.2.0", + release_version="1.2.0", + expected_errors=( + "Release versions differ: Cargo.toml [workspace.package] version is 1.1.9, " + "pyproject.toml [project] version is 1.2.0, " + ".release-please-manifest.json version is 1.2.0.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_diverged_versions_when_checking_then_reports_them( + tmp_path: Path, test_case: NativeVersionTestCase +) -> None: + write_versions(tmp_path, test_case) + + assert get_native_version_errors(tmp_path) == test_case.expected_errors + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryLayeringTestCase( + description="repository release versions agree", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_when_checking_versions_then_they_agree( + test_case: RepositoryLayeringTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[5] + + assert get_native_version_errors(repository) == test_case.expected_errors + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py index 7816e885b5..d41a092202 100644 --- a/tests/unit/src/sqlbuild/lint/helpers.py +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -47,7 +47,7 @@ def write_fixture_format_project( _RESERVED_KEYWORDS_DIR: Path = ( Path(__file__).resolve().parents[5] / "crates" - / "sqlbuild-rules-native" + / "sqlbuild-analysis" / "src" / "sql_tokens" / "reserved_keywords"